From 8beaa388c619df4602f4a8e01cb617ac6bb514a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Pelletier?= Date: Mon, 18 Feb 2019 16:37:38 -0500 Subject: [PATCH 001/667] Added ResourceConfig class --- ScoutSuite/providers/base/configs/resource_config.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 ScoutSuite/providers/base/configs/resource_config.py diff --git a/ScoutSuite/providers/base/configs/resource_config.py b/ScoutSuite/providers/base/configs/resource_config.py new file mode 100644 index 000000000..cc38fe3b4 --- /dev/null +++ b/ScoutSuite/providers/base/configs/resource_config.py @@ -0,0 +1,7 @@ + +class ResourceConfig(object): + def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): + raise NotImplementedError() + + def finalize(self): + raise NotImplementedError() \ No newline at end of file From d0a9dbb9d60e5432ef15b0db44dd7352d3bf2a4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Pelletier?= Date: Mon, 18 Feb 2019 16:41:20 -0500 Subject: [PATCH 002/667] Removed NotImplementedError for finalize() --- ScoutSuite/providers/base/configs/resource_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/base/configs/resource_config.py b/ScoutSuite/providers/base/configs/resource_config.py index cc38fe3b4..49b474bc7 100644 --- a/ScoutSuite/providers/base/configs/resource_config.py +++ b/ScoutSuite/providers/base/configs/resource_config.py @@ -4,4 +4,4 @@ def fetch_all(self, credentials, regions=None, partition_name='aws', targets=Non raise NotImplementedError() def finalize(self): - raise NotImplementedError() \ No newline at end of file + pass \ No newline at end of file From ae94121277c0867999123e6e95aafdd01620d255 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 19 Feb 2019 09:52:33 -0500 Subject: [PATCH 003/667] Made fetch_all and fetch asynchronous --- Scout.py | 7 +++++-- ScoutSuite/__main__.py | 4 ++-- ScoutSuite/providers/base/configs/resource_config.py | 8 ++++++-- ScoutSuite/providers/base/configs/services.py | 4 ++-- ScoutSuite/providers/base/provider.py | 4 ++-- 5 files changed, 17 insertions(+), 10 deletions(-) diff --git a/Scout.py b/Scout.py index 5edb1c55f..253a68e82 100755 --- a/Scout.py +++ b/Scout.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- import sys +import asyncio from ScoutSuite.__main__ import main as scout from ScoutSuite.cli_parser import ScoutSuiteArgumentParser @@ -9,5 +10,7 @@ if __name__ == "__main__": parser = ScoutSuiteArgumentParser() args = parser.parse_args() - - sys.exit(scout(args)) + loop = asyncio.get_event_loop() + loop.run_until_complete(scout(args)) + loop.close() + sys.exit() diff --git a/ScoutSuite/__main__.py b/ScoutSuite/__main__.py index 1acc5d39f..76301c389 100644 --- a/ScoutSuite/__main__.py +++ b/ScoutSuite/__main__.py @@ -17,7 +17,7 @@ from ScoutSuite.providers import get_provider -def main(args): +async def main(args): """ Main method that runs a scan @@ -77,7 +77,7 @@ def main(args): # Fetch data from provider APIs try: - cloud_provider.fetch(regions=args.get('regions')) + await cloud_provider.fetch(regions=args.get('regions')) except KeyboardInterrupt: printInfo('\nCancelled by user') return 130 diff --git a/ScoutSuite/providers/base/configs/resource_config.py b/ScoutSuite/providers/base/configs/resource_config.py index 49b474bc7..089d7c3eb 100644 --- a/ScoutSuite/providers/base/configs/resource_config.py +++ b/ScoutSuite/providers/base/configs/resource_config.py @@ -1,7 +1,11 @@ class ResourceConfig(object): - def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): + + def __init__(self, thread_config=4): + self.thread_config = thread_config + + async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): raise NotImplementedError() def finalize(self): - pass \ No newline at end of file + pass diff --git a/ScoutSuite/providers/base/configs/services.py b/ScoutSuite/providers/base/configs/services.py index 6fcf89400..73227a107 100644 --- a/ScoutSuite/providers/base/configs/services.py +++ b/ScoutSuite/providers/base/configs/services.py @@ -11,7 +11,7 @@ def __init__(self, metadata=None, thread_config=4): def _is_provider(self, provider_name): return False - def fetch(self, credentials, services=None, regions=None): + async def fetch(self, credentials, services=None, regions=None): services = [] if services is None else services regions = [] if regions is None else regions for service in vars(self): @@ -30,7 +30,7 @@ def fetch(self, credentials, services=None, regions=None): if service != 'iam': method_args['partition_name'] = get_partition_name(credentials) - service_config.fetch_all(**method_args) + await service_config.fetch_all(**method_args) if hasattr(service_config, 'finalize'): service_config.finalize() else: diff --git a/ScoutSuite/providers/base/provider.py b/ScoutSuite/providers/base/provider.py index 80cba7087..9db81224b 100644 --- a/ScoutSuite/providers/base/provider.py +++ b/ScoutSuite/providers/base/provider.py @@ -78,7 +78,7 @@ def postprocessing(self, current_time, ruleset): self._update_metadata() self._update_last_run(current_time, ruleset) - def fetch(self, regions=None, skipped_regions=None, partition_name=None): + async def fetch(self, regions=None, skipped_regions=None, partition_name=None): """ Fetch resources for each service @@ -92,7 +92,7 @@ def fetch(self, regions=None, skipped_regions=None, partition_name=None): regions = [] if regions is None else regions skipped_regions = [] if skipped_regions is None else skipped_regions # TODO: determine partition name based on regions and warn if multiple partitions... - self.services.fetch(self.credentials, self.service_list, regions) + await self.services.fetch(self.credentials, self.service_list, regions) # TODO implement this properly """ From 63fc5f6d59fc60392f6b319db99018d74d7cab0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Tue, 19 Feb 2019 16:52:01 -0500 Subject: [PATCH 004/667] Add support for (database) auditing policies resources. --- .../sql/database_blob_auditing_policies.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py diff --git a/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py b/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py new file mode 100644 index 000000000..1f1c1762a --- /dev/null +++ b/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +from azure.mgmt.sql import SqlManagementClient + +from ScoutSuite.providers.base.configs.resource_config import ResourceConfig + + +class DatabaseBlobAuditingPoliciesConfig(ResourceConfig): + + def __init__(self, resource_group_name, server_name, database_name): + self.resource_group_name = resource_group_name + self.server_name = server_name + self.database_name = database_name + + self.value = None + + async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): + # sdk container: + api = SqlManagementClient(credentials.credentials, credentials.subscription_id) + + self.value =\ + api.database_blob_auditing_policies.get(self.resource_group_name, self.server_name, self.database_name) + # TODO: await api.database_blob_auditing_policies.get(self.resource_group_name, self.server_name, self.database_name) From 7927ac3b94010926f02fb37f46a8e54204bedff6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Tue, 19 Feb 2019 16:52:26 -0500 Subject: [PATCH 005/667] Add support for (database) threat detection policy resources. --- .../sql/database_threat_detection_policies.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py diff --git a/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py b/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py new file mode 100644 index 000000000..536bbd420 --- /dev/null +++ b/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +from azure.mgmt.sql import SqlManagementClient + +from ScoutSuite.providers.base.configs.resource_config import ResourceConfig + + +class DatabaseThreatDetectionPoliciesConfig(ResourceConfig): + + def __init__(self, resource_group_name, server_name, database_name): + self.resource_group_name = resource_group_name + self.server_name = server_name + self.database_name = database_name + + self.value = None + + async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): + # sdk container: + api = SqlManagementClient(credentials.credentials, credentials.subscription_id) + + self.value =\ + api.database_threat_detection_policies.get(self.resource_group_name, self.server_name, self.database_name) + # TODO: await api.transparent_data_encryptions.get(self.resource_group_name, self.server_name, self.database_name) From 2b131a8075b9c5a6dce04b309c31f04d13a52196 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Tue, 19 Feb 2019 16:52:46 -0500 Subject: [PATCH 006/667] Add support for (database) replication link resources. --- .../azure/resources/sql/replication_links.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 ScoutSuite/providers/azure/resources/sql/replication_links.py diff --git a/ScoutSuite/providers/azure/resources/sql/replication_links.py b/ScoutSuite/providers/azure/resources/sql/replication_links.py new file mode 100644 index 000000000..1005f0f17 --- /dev/null +++ b/ScoutSuite/providers/azure/resources/sql/replication_links.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +from azure.mgmt.sql import SqlManagementClient + +from ScoutSuite.providers.base.configs.resource_config import ResourceConfig + + +class ReplicationLinksConfig(ResourceConfig): + + def __init__(self, resource_group_name, server_name, database_name): + self.resource_group_name = resource_group_name + self.server_name = server_name + self.database_name = database_name + + self.value = None + + async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): + # sdk container: + api = SqlManagementClient(credentials.credentials, credentials.subscription_id) + + self.value =\ + list(api.replication_links.list_by_database(self.resource_group_name, self.server_name, self.database_name)) + # TODO: await api.replication_links.list_by_databases(self.resource_group_name, self.server_name, self.database_name) From 44f90e9dfb602c06f2b3fa840830ac1ef27d226f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Tue, 19 Feb 2019 16:53:22 -0500 Subject: [PATCH 007/667] Add support for azure ad administrator resources. --- .../sql/server_azure_ad_administrators.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py diff --git a/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py b/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py new file mode 100644 index 000000000..f68f91e9d --- /dev/null +++ b/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- + +from azure.mgmt.sql import SqlManagementClient +from msrestazure.azure_exceptions import CloudError + +from ScoutSuite.providers.base.configs.resource_config import ResourceConfig + + +class ServerAzureAdAdministratorsConfig(ResourceConfig): + + def __init__(self, resource_group_name, server_name): + self.resource_group_name = resource_group_name + self.server_name = server_name + + self.value = None + + async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): + # sdk container: + api = SqlManagementClient(credentials.credentials, credentials.subscription_id) + + try: + self.value = \ + api.server_azure_ad_administrators.get(self.resource_group_name, self.server_name) + # TODO: await api.server_azure_ad_administrators.get(self.resource_group_name, self.server_name) + except CloudError: # no ad admin configured returns a 404 error + pass From 381a82f7d569409a4612229be0b06d875d5eb5bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Tue, 19 Feb 2019 16:53:46 -0500 Subject: [PATCH 008/667] Add support for (database) transparent data encryption resources. --- .../sql/transparent_data_encryptions.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py diff --git a/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py b/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py new file mode 100644 index 000000000..e2a582046 --- /dev/null +++ b/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- + +from azure.mgmt.sql import SqlManagementClient + +from ScoutSuite.providers.base.configs.resource_config import ResourceConfig + + +class TransparentDataEncryptionsConfig(ResourceConfig): + + def __init__(self, resource_group_name, server_name, database_name): + self.resource_group_name = resource_group_name + self.server_name = server_name + self.database_name = database_name + + self.value = None + + async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): + # sdk container: + api = SqlManagementClient(credentials.credentials, credentials.subscription_id) + + self.value =\ + api.transparent_data_encryptions.get(self.resource_group_name, self.server_name, self.database_name) + # TODO: await api.transparent_data_encryptions.get(self.resource_group_name, self.server_name, self.database_name) + From b13f22419dd26c0929d84449a69b709938cfd378 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Tue, 19 Feb 2019 16:55:28 -0500 Subject: [PATCH 009/667] Remove thread_config and therefore init method. --- ScoutSuite/providers/base/configs/resource_config.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/ScoutSuite/providers/base/configs/resource_config.py b/ScoutSuite/providers/base/configs/resource_config.py index 089d7c3eb..4f74c54b2 100644 --- a/ScoutSuite/providers/base/configs/resource_config.py +++ b/ScoutSuite/providers/base/configs/resource_config.py @@ -1,9 +1,6 @@ class ResourceConfig(object): - def __init__(self, thread_config=4): - self.thread_config = thread_config - async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): raise NotImplementedError() From cb4c57c6199247ed3a43fbf78c25573f685571cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Tue, 19 Feb 2019 16:55:55 -0500 Subject: [PATCH 010/667] Add __init__ to python packages. --- ScoutSuite/providers/azure/resources/__init__.py | 0 ScoutSuite/providers/azure/resources/sql/__init__.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 ScoutSuite/providers/azure/resources/__init__.py create mode 100644 ScoutSuite/providers/azure/resources/sql/__init__.py diff --git a/ScoutSuite/providers/azure/resources/__init__.py b/ScoutSuite/providers/azure/resources/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ScoutSuite/providers/azure/resources/sql/__init__.py b/ScoutSuite/providers/azure/resources/sql/__init__.py new file mode 100644 index 000000000..e69de29bb From 0efcf1b85440ce4ad550e15702b2aa9d32b2b02e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Tue, 19 Feb 2019 16:56:45 -0500 Subject: [PATCH 011/667] Add get_non_provider_id utility. --- ScoutSuite/providers/azure/resources/utils.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 ScoutSuite/providers/azure/resources/utils.py diff --git a/ScoutSuite/providers/azure/resources/utils.py b/ScoutSuite/providers/azure/resources/utils.py new file mode 100644 index 000000000..3011743f7 --- /dev/null +++ b/ScoutSuite/providers/azure/resources/utils.py @@ -0,0 +1,14 @@ +from hashlib import sha1 + + +def get_non_provider_id(name): + """ + Not all resources have an ID and some services allow the use of "." in names, which break's Scout2's + recursion scheme if name is used as an ID. Use SHA1(name) instead. + + :param name: Name of the resource to + :return: SHA1(name) + """ + m = sha1() + m.update(name.encode('utf-8')) + return m.hexdigest() From 7837b3897119263ddf4ba696353d799e15b1ef82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Tue, 19 Feb 2019 16:57:33 -0500 Subject: [PATCH 012/667] Add support for (sql) database resources. --- .../azure/resources/sql/databases.py | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 ScoutSuite/providers/azure/resources/sql/databases.py diff --git a/ScoutSuite/providers/azure/resources/sql/databases.py b/ScoutSuite/providers/azure/resources/sql/databases.py new file mode 100644 index 000000000..21ac1eb0d --- /dev/null +++ b/ScoutSuite/providers/azure/resources/sql/databases.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- + +from azure.mgmt.sql import SqlManagementClient + +from ScoutSuite.providers.base.configs.resource_config import ResourceConfig + +from .database_blob_auditing_policies import DatabaseBlobAuditingPoliciesConfig +from .database_threat_detection_policies import DatabaseThreatDetectionPoliciesConfig +from .replication_links import ReplicationLinksConfig +from .transparent_data_encryptions import TransparentDataEncryptionsConfig + + +class DatabasesConfig(ResourceConfig): + # the following static methods will be used to parse the config of each resource nested in a database config + # (defined in 'children' attribute below): + @staticmethod + def _parse_auditing_config(config): + return { + 'auditing_enabled': config.value.state == "Enabled" + } + + @staticmethod + def _parse_threat_detection_config(config): + return { + 'threat_detection_enabled': config.value.state == "Enabled" + } + + @staticmethod + def _parse_replication_links_config(config): + return { + 'replication_configured': len(config.value) > 0 + } + + @staticmethod + def _parse_transparent_data_encryption_config(config): + return { + 'transparent_data_encryption_enabled': config.value.status == "Enabled" + } + + # register children resources: + children = { + 'Database Auditing Settings': (DatabaseBlobAuditingPoliciesConfig, _parse_auditing_config), + 'Database Threat Detection Policies': (DatabaseThreatDetectionPoliciesConfig, _parse_threat_detection_config), + 'Replication Links': (ReplicationLinksConfig, _parse_replication_links_config), + 'Transparent Data Encryptions': (TransparentDataEncryptionsConfig, _parse_transparent_data_encryption_config) + } + + def __init__(self, resource_group_name, server_name): + self.resource_group_name = resource_group_name + self.server_name = server_name + + self.databases = {} + + async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): + # sdk container: + api = SqlManagementClient(credentials.credentials, credentials.subscription_id) + + # TODO: for db in await api.databases.list_by_server(self.resource_group_name, self.server_name): + for db in api.databases.list_by_server(self.resource_group_name, self.server_name): + self.databases[db.name] = { + 'id': db.name, + } + + # put the following code in a fetch_children() parent method (typical method for a composite node)? + for (resource_config, resource_parser) in self.children.values(): + resource = resource_config(self.resource_group_name, self.server_name, db.name) + await resource.fetch_all(credentials) + for k, v in resource_parser.__func__(resource).items(): + self.databases[db.name][k] = v From af1cda2b07ff88524bb21c4c1fcca4d9517cb0f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Tue, 19 Feb 2019 16:57:44 -0500 Subject: [PATCH 013/667] Add support for (sql) server resources. --- .../providers/azure/resources/sql/servers.py | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 ScoutSuite/providers/azure/resources/sql/servers.py diff --git a/ScoutSuite/providers/azure/resources/sql/servers.py b/ScoutSuite/providers/azure/resources/sql/servers.py new file mode 100644 index 000000000..f1cc2b572 --- /dev/null +++ b/ScoutSuite/providers/azure/resources/sql/servers.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- + +from azure.mgmt.sql import SqlManagementClient + +from ScoutSuite.providers.base.configs.resource_config import ResourceConfig +from ScoutSuite.providers.azure.utils import get_resource_group_name + +from .databases import DatabasesConfig +from .server_azure_ad_administrators import ServerAzureAdAdministratorsConfig +from ..utils import get_non_provider_id + + +class ServersConfig(ResourceConfig): + # the following static methods will be used to parse the config of each resource nested in a server config + # (defined in 'children' attribute below): + @staticmethod + def _parse_databases_config(config): + return { + 'databases': config.databases + } + + @staticmethod + def _parse_azure_ad_administrators_config(config): + return { + 'ad_admin_configured': config.value is not None + } + + # register children resources: + children = { + 'Databases': (DatabasesConfig, _parse_databases_config), + 'Azure AD Administrators': (ServerAzureAdAdministratorsConfig, _parse_azure_ad_administrators_config) + } + + def __init__(self): + self.servers = {} + + async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): + # sdk container: + api = SqlManagementClient(credentials.credentials, credentials.subscription_id) + + # for server in await api.servers.list(): + for server in api.servers.list(): + # parsing: + + id = get_non_provider_id(server.id) + resource_group_name = get_resource_group_name(server.id) + + self.servers[id] = { + 'id': id, + 'name': server.name + } + + # put the following code in a fetch_children() parent method (typical method for a composite node)? + for (resource_config, resource_parser) in self.children.values(): + resource = resource_config(resource_group_name, server.name) + await resource.fetch_all(credentials) + for k, v in resource_parser.__func__(resource).items(): + self.servers[id][k] = v From 1a1762c4683cda48542c1057d29a0c5954bc047f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Tue, 19 Feb 2019 16:58:46 -0500 Subject: [PATCH 014/667] Switch SQLDatabase service implementation to the new one. --- ScoutSuite/providers/azure/configs/services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/azure/configs/services.py b/ScoutSuite/providers/azure/configs/services.py index 7bbaa1b13..793fb2ade 100644 --- a/ScoutSuite/providers/azure/configs/services.py +++ b/ScoutSuite/providers/azure/configs/services.py @@ -3,7 +3,7 @@ from ScoutSuite.providers.base.configs.services import BaseServicesConfig from ScoutSuite.providers.azure.services.storageaccounts import StorageAccountsConfig from ScoutSuite.providers.azure.services.monitor import MonitorConfig -from ScoutSuite.providers.azure.services.sqldatabase import SQLDatabaseConfig +from ScoutSuite.providers.azure.resources.sql.servers import ServersConfig as SQLDatabaseConfig from ScoutSuite.providers.azure.services.securitycenter import SecurityCenterConfig from ScoutSuite.providers.azure.services.network import NetworkConfig from ScoutSuite.providers.azure.services.keyvault import KeyVaultConfig @@ -31,7 +31,7 @@ def __init__(self, metadata=None, thread_config=4, **kwargs): self.storageaccounts = StorageAccountsConfig(thread_config=thread_config) self.monitor = MonitorConfig(thread_config=thread_config) - self.sqldatabase = SQLDatabaseConfig(thread_config=thread_config) + self.sqldatabase = SQLDatabaseConfig() self.securitycenter = SecurityCenterConfig(thread_config=thread_config) self.network = NetworkConfig(thread_config=thread_config) self.keyvault = KeyVaultConfig(thread_config=thread_config) From 5860417d4f1bcccf5da1ec42df2b096db2f90d05 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 20 Feb 2019 11:15:31 -0500 Subject: [PATCH 015/667] Made finalize async --- ScoutSuite/providers/base/configs/resource_config.py | 2 +- ScoutSuite/providers/base/configs/services.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/base/configs/resource_config.py b/ScoutSuite/providers/base/configs/resource_config.py index 089d7c3eb..e156ecbe4 100644 --- a/ScoutSuite/providers/base/configs/resource_config.py +++ b/ScoutSuite/providers/base/configs/resource_config.py @@ -7,5 +7,5 @@ def __init__(self, thread_config=4): async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): raise NotImplementedError() - def finalize(self): + async def finalize(self): pass diff --git a/ScoutSuite/providers/base/configs/services.py b/ScoutSuite/providers/base/configs/services.py index 73227a107..f5aa77611 100644 --- a/ScoutSuite/providers/base/configs/services.py +++ b/ScoutSuite/providers/base/configs/services.py @@ -32,7 +32,7 @@ async def fetch(self, credentials, services=None, regions=None): await service_config.fetch_all(**method_args) if hasattr(service_config, 'finalize'): - service_config.finalize() + await service_config.finalize() else: printDebug('No method to fetch service %s.' % service) except Exception as e: From d38e3b7c64cfd72b34c76fa331c0d33fd4033d50 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 20 Feb 2019 11:16:48 -0500 Subject: [PATCH 016/667] Renamed thread_config to _thread_config, since it's private --- ScoutSuite/providers/base/configs/resource_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/base/configs/resource_config.py b/ScoutSuite/providers/base/configs/resource_config.py index e156ecbe4..25d1499de 100644 --- a/ScoutSuite/providers/base/configs/resource_config.py +++ b/ScoutSuite/providers/base/configs/resource_config.py @@ -2,7 +2,7 @@ class ResourceConfig(object): def __init__(self, thread_config=4): - self.thread_config = thread_config + self._thread_config = thread_config async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): raise NotImplementedError() From e5a81ccb69121e14c68799ef91b559f0da66055c Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 20 Feb 2019 11:18:09 -0500 Subject: [PATCH 017/667] Made ResourceConfig inherit from dict to solve the three Ps from hell: private properties problem --- ScoutSuite/providers/base/configs/resource_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/base/configs/resource_config.py b/ScoutSuite/providers/base/configs/resource_config.py index 25d1499de..8eb1dfae8 100644 --- a/ScoutSuite/providers/base/configs/resource_config.py +++ b/ScoutSuite/providers/base/configs/resource_config.py @@ -1,5 +1,5 @@ -class ResourceConfig(object): +class ResourceConfig(dict): def __init__(self, thread_config=4): self._thread_config = thread_config From 2b69395e1e9175ce270c8e8d61abfd5ff2b7c50c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Wed, 20 Feb 2019 14:58:11 -0500 Subject: [PATCH 018/667] Remove useless keys and make children a list of tuples. --- .../providers/azure/resources/sql/databases.py | 14 +++++++------- .../providers/azure/resources/sql/servers.py | 10 +++++----- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/databases.py b/ScoutSuite/providers/azure/resources/sql/databases.py index 21ac1eb0d..732aabfdf 100644 --- a/ScoutSuite/providers/azure/resources/sql/databases.py +++ b/ScoutSuite/providers/azure/resources/sql/databases.py @@ -38,12 +38,12 @@ def _parse_transparent_data_encryption_config(config): } # register children resources: - children = { - 'Database Auditing Settings': (DatabaseBlobAuditingPoliciesConfig, _parse_auditing_config), - 'Database Threat Detection Policies': (DatabaseThreatDetectionPoliciesConfig, _parse_threat_detection_config), - 'Replication Links': (ReplicationLinksConfig, _parse_replication_links_config), - 'Transparent Data Encryptions': (TransparentDataEncryptionsConfig, _parse_transparent_data_encryption_config) - } + children = [ + (DatabaseBlobAuditingPoliciesConfig, _parse_auditing_config), + (DatabaseThreatDetectionPoliciesConfig, _parse_threat_detection_config), + (ReplicationLinksConfig, _parse_replication_links_config), + (TransparentDataEncryptionsConfig, _parse_transparent_data_encryption_config) + ] def __init__(self, resource_group_name, server_name): self.resource_group_name = resource_group_name @@ -62,7 +62,7 @@ async def fetch_all(self, credentials, regions=None, partition_name='aws', targe } # put the following code in a fetch_children() parent method (typical method for a composite node)? - for (resource_config, resource_parser) in self.children.values(): + for (resource_config, resource_parser) in self.children: resource = resource_config(self.resource_group_name, self.server_name, db.name) await resource.fetch_all(credentials) for k, v in resource_parser.__func__(resource).items(): diff --git a/ScoutSuite/providers/azure/resources/sql/servers.py b/ScoutSuite/providers/azure/resources/sql/servers.py index f1cc2b572..a44b913c9 100644 --- a/ScoutSuite/providers/azure/resources/sql/servers.py +++ b/ScoutSuite/providers/azure/resources/sql/servers.py @@ -26,10 +26,10 @@ def _parse_azure_ad_administrators_config(config): } # register children resources: - children = { - 'Databases': (DatabasesConfig, _parse_databases_config), - 'Azure AD Administrators': (ServerAzureAdAdministratorsConfig, _parse_azure_ad_administrators_config) - } + children = [ + (DatabasesConfig, _parse_databases_config), + (ServerAzureAdAdministratorsConfig, _parse_azure_ad_administrators_config) + ] def __init__(self): self.servers = {} @@ -51,7 +51,7 @@ async def fetch_all(self, credentials, regions=None, partition_name='aws', targe } # put the following code in a fetch_children() parent method (typical method for a composite node)? - for (resource_config, resource_parser) in self.children.values(): + for (resource_config, resource_parser) in self.children: resource = resource_config(resource_group_name, server.name) await resource.fetch_all(credentials) for k, v in resource_parser.__func__(resource).items(): From 76372ea333f4eabc7491e9e51ecafd0dbc23a78f Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 20 Feb 2019 15:07:51 -0500 Subject: [PATCH 019/667] Removed thread config --- ScoutSuite/providers/base/configs/resource_config.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/ScoutSuite/providers/base/configs/resource_config.py b/ScoutSuite/providers/base/configs/resource_config.py index 8eb1dfae8..bce48ac3a 100644 --- a/ScoutSuite/providers/base/configs/resource_config.py +++ b/ScoutSuite/providers/base/configs/resource_config.py @@ -1,9 +1,6 @@ class ResourceConfig(dict): - def __init__(self, thread_config=4): - self._thread_config = thread_config - async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): raise NotImplementedError() From 021854f6ad3f60819faa7c337cb10b82321cc41d Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 20 Feb 2019 15:08:12 -0500 Subject: [PATCH 020/667] Implemented aws facade --- ScoutSuite/providers/aws/aws_facade.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 ScoutSuite/providers/aws/aws_facade.py diff --git a/ScoutSuite/providers/aws/aws_facade.py b/ScoutSuite/providers/aws/aws_facade.py new file mode 100644 index 000000000..13eb5ccd8 --- /dev/null +++ b/ScoutSuite/providers/aws/aws_facade.py @@ -0,0 +1,18 @@ +import boto3 + +class AwsFacade(object): + def get_lambda_functions(self, region): + aws_lambda = boto3.client('lambda', region_name=region) + + functions = [] + + marker = None + while True: + response = aws_lambda.list_functions() + + functions.extend(response['Functions']) + marker = response.get('NextMarker', None) + if marker is None: + break + + return functions \ No newline at end of file From 82eb22fad4f24949c5c782b0e2372fade6aab9b6 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 20 Feb 2019 15:08:26 -0500 Subject: [PATCH 021/667] Implemented regions config --- .../providers/aws/configs/regions_config.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 ScoutSuite/providers/aws/configs/regions_config.py diff --git a/ScoutSuite/providers/aws/configs/regions_config.py b/ScoutSuite/providers/aws/configs/regions_config.py new file mode 100644 index 000000000..672fb4dba --- /dev/null +++ b/ScoutSuite/providers/aws/configs/regions_config.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +from ScoutSuite.providers.base.configs.resource_config import ResourceConfig +from botocore.session import Session +from collections import Counter + + +class RegionsConfig(ResourceConfig): + + def __init__(self, service, child_config_type): + self._service = service + self._child_config_type = child_config_type + + async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): + for region in await self._build_region_list(regions, partition_name): + child = self._child_config_type() + await child.fetch_all(credentials, region, partition_name, None) + self.setdefault(region, child) + + async def _build_region_list(self, chosen_regions=None, partition_name='aws'): + service = 'ec2containerservice' if self._service == 'ecs' else self._service + available_services = Session().get_available_services() + + if service not in available_services: + raise Exception('Service ' + service + ' is not available.') + + regions = Session().get_available_regions(service, partition_name) + + if chosen_regions: + return list((Counter(regions) & Counter(chosen_regions)).elements()) + else: + return regions From 5c2a02a3ee562c0c60e4b6df80929ffc77de73bb Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 20 Feb 2019 15:09:03 -0500 Subject: [PATCH 022/667] Migrated aws lambda to new architecture --- ScoutSuite/providers/aws/configs/services.py | 4 +- .../providers/aws/services/awslambda.py | 60 +++++++++---------- 2 files changed, 30 insertions(+), 34 deletions(-) diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index 0f7ec191c..b6e35566e 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -2,7 +2,7 @@ from opinel.utils.console import printError, printException, printInfo, printDebug -from ScoutSuite.providers.aws.services.awslambda import LambdaConfig +from ScoutSuite.providers.aws.services.awslambda import LambdaServiceConfig from ScoutSuite.providers.aws.services.cloudformation import CloudFormationConfig from ScoutSuite.providers.aws.services.cloudtrail import CloudTrailConfig from ScoutSuite.providers.aws.services.cloudwatch import CloudWatchConfig @@ -64,7 +64,7 @@ def __init__(self, metadata=None, thread_config=4, **kwargs): self.emr = EMRConfig(metadata['analytics']['emr'], thread_config) self.iam = IAMConfig(thread_config) self.kms = KMSConfig(metadata['security']['kms'], thread_config) - self.awslambda = LambdaConfig(metadata['compute']['awslambda'], thread_config) + self.awslambda = LambdaServiceConfig() self.redshift = RedshiftConfig(metadata['database']['redshift'], thread_config) self.rds = RDSConfig(metadata['database']['rds'], thread_config) self.route53 = Route53Config(thread_config) diff --git a/ScoutSuite/providers/aws/services/awslambda.py b/ScoutSuite/providers/aws/services/awslambda.py index bb567c5ae..c633b1016 100644 --- a/ScoutSuite/providers/aws/services/awslambda.py +++ b/ScoutSuite/providers/aws/services/awslambda.py @@ -1,34 +1,30 @@ # -*- coding: utf-8 -*- -""" -Lambda-related classes and functions -""" - -from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients - - - -######################################## -# LambdaRegionConfig -######################################## - -class LambdaRegionConfig(RegionConfig): - - def parse_function(self, global_params, region, function): +from ScoutSuite.providers.base.configs.resource_config import ResourceConfig +from ScoutSuite.providers.aws.configs.regions_config import RegionsConfig +from ScoutSuite.providers.aws.aws_facade import AwsFacade +from opinel.utils.aws import build_region_list + +class LambdaServiceConfig(ResourceConfig): + async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): + regions = RegionsConfig('lambda', LambdasConfig) + await regions.fetch_all(credentials, regions, partition_name, None) + self.setdefault('regions_count', len(regions)) + self.setdefault('regions', regions) + +class LambdasConfig(ResourceConfig): + async def fetch_all(self, credentials, region, partition_name='aws', targets=None): + # TODO: Should be injected + facade = AwsFacade() + + functions = {} + for raw_function in facade.get_lambda_functions(region): + name, function = self.parse_function(raw_function) + functions[name] = function + + self.setdefault('functions_count', len(functions)) + self.setdefault('functions', functions) + + @staticmethod + def parse_function(function): function['name'] = function.pop('FunctionName') - self.functions[function['name']] = function - - - -######################################## -# LambdaConfig -######################################## - -class LambdaConfig(RegionalServiceConfig): - """ - Lambda configuration for all AWS regions - """ - - region_config_class = LambdaRegionConfig - - def __init__(self, service_metadata, thread_config = 4): - super(LambdaConfig, self).__init__(service_metadata, thread_config) + return (function['name'], function) From ce9e9c193a06138c8b5d03f8a865a8f5ef9b847e Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 20 Feb 2019 15:13:44 -0500 Subject: [PATCH 023/667] Moved build region list to aws facade --- ScoutSuite/providers/aws/aws_facade.py | 18 ++++++++++++++- .../providers/aws/configs/regions_config.py | 22 +++++-------------- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/ScoutSuite/providers/aws/aws_facade.py b/ScoutSuite/providers/aws/aws_facade.py index 13eb5ccd8..c49f01d88 100644 --- a/ScoutSuite/providers/aws/aws_facade.py +++ b/ScoutSuite/providers/aws/aws_facade.py @@ -1,4 +1,6 @@ import boto3 +from botocore.session import Session +from collections import Counter class AwsFacade(object): def get_lambda_functions(self, region): @@ -15,4 +17,18 @@ def get_lambda_functions(self, region): if marker is None: break - return functions \ No newline at end of file + return functions + + async def build_region_list(self, service, chosen_regions=None, partition_name='aws'): + service = 'ec2containerservice' if service == 'ecs' else service + available_services = Session().get_available_services() + + if service not in available_services: + raise Exception('Service ' + service + ' is not available.') + + regions = Session().get_available_regions(service, partition_name) + + if chosen_regions: + return list((Counter(regions) & Counter(chosen_regions)).elements()) + else: + return regions \ No newline at end of file diff --git a/ScoutSuite/providers/aws/configs/regions_config.py b/ScoutSuite/providers/aws/configs/regions_config.py index 672fb4dba..2210bc77a 100644 --- a/ScoutSuite/providers/aws/configs/regions_config.py +++ b/ScoutSuite/providers/aws/configs/regions_config.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- from ScoutSuite.providers.base.configs.resource_config import ResourceConfig -from botocore.session import Session -from collections import Counter +from ScoutSuite.providers.aws.aws_facade import AwsFacade class RegionsConfig(ResourceConfig): @@ -11,21 +10,10 @@ def __init__(self, service, child_config_type): self._child_config_type = child_config_type async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): - for region in await self._build_region_list(regions, partition_name): + # TODO: Should be injected + facade = AwsFacade() + + for region in await facade.build_region_list(self._service, regions, partition_name): child = self._child_config_type() await child.fetch_all(credentials, region, partition_name, None) self.setdefault(region, child) - - async def _build_region_list(self, chosen_regions=None, partition_name='aws'): - service = 'ec2containerservice' if self._service == 'ecs' else self._service - available_services = Session().get_available_services() - - if service not in available_services: - raise Exception('Service ' + service + ' is not available.') - - regions = Session().get_available_regions(service, partition_name) - - if chosen_regions: - return list((Counter(regions) & Counter(chosen_regions)).elements()) - else: - return regions From 0f3445afd208d56942f7743537574c7d260e91dc Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 20 Feb 2019 17:49:54 -0500 Subject: [PATCH 024/667] Update aws_facade.py --- ScoutSuite/providers/aws/aws_facade.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/aws_facade.py b/ScoutSuite/providers/aws/aws_facade.py index c49f01d88..06e0dc941 100644 --- a/ScoutSuite/providers/aws/aws_facade.py +++ b/ScoutSuite/providers/aws/aws_facade.py @@ -2,6 +2,7 @@ from botocore.session import Session from collections import Counter +# TODO: Handle authentication better. I don't even know how it currently works. I think connect_service is called somewhere. class AwsFacade(object): def get_lambda_functions(self, region): aws_lambda = boto3.client('lambda', region_name=region) @@ -31,4 +32,4 @@ async def build_region_list(self, service, chosen_regions=None, partition_name=' if chosen_regions: return list((Counter(regions) & Counter(chosen_regions)).elements()) else: - return regions \ No newline at end of file + return regions From d4a22dfccab6504d4b8abca2125b8c93b04a33c2 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 20 Feb 2019 20:12:09 -0500 Subject: [PATCH 025/667] Removed unnecessary line --- ScoutSuite/providers/aws/aws_facade.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ScoutSuite/providers/aws/aws_facade.py b/ScoutSuite/providers/aws/aws_facade.py index 06e0dc941..19272c66a 100644 --- a/ScoutSuite/providers/aws/aws_facade.py +++ b/ScoutSuite/providers/aws/aws_facade.py @@ -9,7 +9,6 @@ def get_lambda_functions(self, region): functions = [] - marker = None while True: response = aws_lambda.list_functions() From 26108c782338a15950fb21e973d6cec3d479bbfa Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 20 Feb 2019 20:42:06 -0500 Subject: [PATCH 026/667] Made the code more pythonic --- ScoutSuite/providers/aws/services/awslambda.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/services/awslambda.py b/ScoutSuite/providers/aws/services/awslambda.py index c633b1016..d1ddf3184 100644 --- a/ScoutSuite/providers/aws/services/awslambda.py +++ b/ScoutSuite/providers/aws/services/awslambda.py @@ -21,7 +21,7 @@ async def fetch_all(self, credentials, region, partition_name='aws', targets=Non name, function = self.parse_function(raw_function) functions[name] = function - self.setdefault('functions_count', len(functions)) + self['functions_count'] = len(functions)) self.setdefault('functions', functions) @staticmethod From f8ac389a31cab7c96c5bdb30460f0b0d376aec77 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 20 Feb 2019 21:20:38 -0500 Subject: [PATCH 027/667] Made RegionConfig a base class --- .../providers/aws/configs/regions_config.py | 14 ++++++------- .../providers/aws/services/awslambda.py | 21 +++++++++++++------ .../providers/base/configs/resource_config.py | 2 +- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/ScoutSuite/providers/aws/configs/regions_config.py b/ScoutSuite/providers/aws/configs/regions_config.py index 2210bc77a..86ebe06f2 100644 --- a/ScoutSuite/providers/aws/configs/regions_config.py +++ b/ScoutSuite/providers/aws/configs/regions_config.py @@ -5,15 +5,15 @@ class RegionsConfig(ResourceConfig): - def __init__(self, service, child_config_type): + def __init__(self, service): self._service = service - self._child_config_type = child_config_type - async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): + async def fetch_all(self, chosen_regions=None, partition_name='aws'): # TODO: Should be injected facade = AwsFacade() - for region in await facade.build_region_list(self._service, regions, partition_name): - child = self._child_config_type() - await child.fetch_all(credentials, region, partition_name, None) - self.setdefault(region, child) + self['regions'] = {} + for region in await facade.build_region_list(self._service, chosen_regions, partition_name): + self['regions'][region] = {} + + self.setdefault('regions_count', len(self['regions'])) diff --git a/ScoutSuite/providers/aws/services/awslambda.py b/ScoutSuite/providers/aws/services/awslambda.py index d1ddf3184..4905e4212 100644 --- a/ScoutSuite/providers/aws/services/awslambda.py +++ b/ScoutSuite/providers/aws/services/awslambda.py @@ -4,15 +4,24 @@ from ScoutSuite.providers.aws.aws_facade import AwsFacade from opinel.utils.aws import build_region_list -class LambdaServiceConfig(ResourceConfig): +class LambdaServiceConfig(RegionsConfig): + def __init__(self): + super(LambdaServiceConfig, self).__init__('lambda') + async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): - regions = RegionsConfig('lambda', LambdasConfig) - await regions.fetch_all(credentials, regions, partition_name, None) - self.setdefault('regions_count', len(regions)) - self.setdefault('regions', regions) + await super(LambdaServiceConfig, self).fetch_all( + chosen_regions=regions, + partition_name=partition_name + ) + + for region in self['regions']: + functions = LambdasConfig() + await functions.fetch_all(region=region) + self['regions'][region] = functions + class LambdasConfig(ResourceConfig): - async def fetch_all(self, credentials, region, partition_name='aws', targets=None): + async def fetch_all(self, region): # TODO: Should be injected facade = AwsFacade() diff --git a/ScoutSuite/providers/base/configs/resource_config.py b/ScoutSuite/providers/base/configs/resource_config.py index bce48ac3a..4f70c8ea1 100644 --- a/ScoutSuite/providers/base/configs/resource_config.py +++ b/ScoutSuite/providers/base/configs/resource_config.py @@ -1,7 +1,7 @@ class ResourceConfig(dict): - async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): + async def fetch_all(self, **kwargs): raise NotImplementedError() async def finalize(self): From efaebfd11613e83caf721edbd8c2b61d949e7eae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Wed, 20 Feb 2019 22:13:17 -0500 Subject: [PATCH 028/667] Rename ResourceConfig class to Resources, make it abstract and make fetch_all method more general. --- ScoutSuite/providers/base/configs/resource_config.py | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 ScoutSuite/providers/base/configs/resource_config.py diff --git a/ScoutSuite/providers/base/configs/resource_config.py b/ScoutSuite/providers/base/configs/resource_config.py deleted file mode 100644 index bce48ac3a..000000000 --- a/ScoutSuite/providers/base/configs/resource_config.py +++ /dev/null @@ -1,8 +0,0 @@ - -class ResourceConfig(dict): - - async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): - raise NotImplementedError() - - async def finalize(self): - pass From 7f7ab96fbfbb9bd54308385423d22b80a4bd56be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sgha=C3=AFer?= Date: Wed, 20 Feb 2019 22:45:52 -0500 Subject: [PATCH 029/667] Used [] instead of set default Co-Authored-By: Aboisier --- ScoutSuite/providers/aws/configs/regions_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/configs/regions_config.py b/ScoutSuite/providers/aws/configs/regions_config.py index 86ebe06f2..9f36ec6c6 100644 --- a/ScoutSuite/providers/aws/configs/regions_config.py +++ b/ScoutSuite/providers/aws/configs/regions_config.py @@ -16,4 +16,4 @@ async def fetch_all(self, chosen_regions=None, partition_name='aws'): for region in await facade.build_region_list(self._service, chosen_regions, partition_name): self['regions'][region] = {} - self.setdefault('regions_count', len(self['regions'])) + self['regions_count'] = len(self['regions']) From 5e9eec1dfe123b7a0bb2a9f2665475f4d0c23dc4 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 20 Feb 2019 22:48:39 -0500 Subject: [PATCH 030/667] Removed unused arguments --- ScoutSuite/providers/aws/services/awslambda.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/services/awslambda.py b/ScoutSuite/providers/aws/services/awslambda.py index 4905e4212..767382b20 100644 --- a/ScoutSuite/providers/aws/services/awslambda.py +++ b/ScoutSuite/providers/aws/services/awslambda.py @@ -8,7 +8,7 @@ class LambdaServiceConfig(RegionsConfig): def __init__(self): super(LambdaServiceConfig, self).__init__('lambda') - async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): + async def fetch_all(self, regions=None, partition_name='aws'): await super(LambdaServiceConfig, self).fetch_all( chosen_regions=regions, partition_name=partition_name From 19d9295ee212a2a8ee9c2411747a5ef697823051 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Wed, 20 Feb 2019 23:45:44 -0500 Subject: [PATCH 031/667] Modify base class and add parsing process. --- .../sql/database_blob_auditing_policies.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py b/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py index 1f1c1762a..743b0e8a2 100644 --- a/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py +++ b/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py @@ -2,22 +2,26 @@ from azure.mgmt.sql import SqlManagementClient -from ScoutSuite.providers.base.configs.resource_config import ResourceConfig +from ScoutSuite.providers.base.configs.resources import Resources -class DatabaseBlobAuditingPoliciesConfig(ResourceConfig): +class DatabaseBlobAuditingPolicies(Resources): def __init__(self, resource_group_name, server_name, database_name): self.resource_group_name = resource_group_name self.server_name = server_name self.database_name = database_name - self.value = None - - async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): + async def fetch_all(self, credentials): # sdk container: api = SqlManagementClient(credentials.credentials, credentials.subscription_id) - self.value =\ + policies =\ api.database_blob_auditing_policies.get(self.resource_group_name, self.server_name, self.database_name) # TODO: await api.database_blob_auditing_policies.get(self.resource_group_name, self.server_name, self.database_name) + + self['auditing_enabled'] = self._is_auditing_enabled(policies) + + @staticmethod + def _is_auditing_enabled(policies): + return policies.state == "Enabled" From 7af989e0579c5d570ea58ab2b1557bb934fdb1a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Wed, 20 Feb 2019 23:45:53 -0500 Subject: [PATCH 032/667] Modify base class and add parsing process. --- .../sql/database_threat_detection_policies.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py b/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py index 536bbd420..874b14307 100644 --- a/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py +++ b/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py @@ -2,22 +2,26 @@ from azure.mgmt.sql import SqlManagementClient -from ScoutSuite.providers.base.configs.resource_config import ResourceConfig +from ScoutSuite.providers.base.configs.resources import Resources -class DatabaseThreatDetectionPoliciesConfig(ResourceConfig): +class DatabaseThreatDetectionPolicies(Resources): def __init__(self, resource_group_name, server_name, database_name): self.resource_group_name = resource_group_name self.server_name = server_name self.database_name = database_name - self.value = None - - async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): + async def fetch_all(self, credentials): # sdk container: api = SqlManagementClient(credentials.credentials, credentials.subscription_id) - self.value =\ + policies =\ api.database_threat_detection_policies.get(self.resource_group_name, self.server_name, self.database_name) # TODO: await api.transparent_data_encryptions.get(self.resource_group_name, self.server_name, self.database_name) + + self['threat_detection_enabled'] = self._is_threat_detection_enabled(policies) + + @staticmethod + def _is_threat_detection_enabled(policies): + return policies.state == "Enabled" From df2973f067a964730213fad521b733bb23c1f3ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Wed, 20 Feb 2019 23:46:04 -0500 Subject: [PATCH 033/667] Modify base class and add parsing process. --- .../azure/resources/sql/replication_links.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/replication_links.py b/ScoutSuite/providers/azure/resources/sql/replication_links.py index 1005f0f17..20db831f9 100644 --- a/ScoutSuite/providers/azure/resources/sql/replication_links.py +++ b/ScoutSuite/providers/azure/resources/sql/replication_links.py @@ -2,22 +2,26 @@ from azure.mgmt.sql import SqlManagementClient -from ScoutSuite.providers.base.configs.resource_config import ResourceConfig +from ScoutSuite.providers.base.configs.resources import Resources -class ReplicationLinksConfig(ResourceConfig): +class ReplicationLinks(Resources): def __init__(self, resource_group_name, server_name, database_name): self.resource_group_name = resource_group_name self.server_name = server_name self.database_name = database_name - self.value = None - async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): # sdk container: api = SqlManagementClient(credentials.credentials, credentials.subscription_id) - self.value =\ + links =\ list(api.replication_links.list_by_database(self.resource_group_name, self.server_name, self.database_name)) # TODO: await api.replication_links.list_by_databases(self.resource_group_name, self.server_name, self.database_name) + + self['replication_configured'] = self._is_replication_configured(links) + + @staticmethod + def _is_replication_configured(links): + return len(links) > 0 From 89a7718bea59c4783e8f8d587cf8ace7973f82e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Wed, 20 Feb 2019 23:46:16 -0500 Subject: [PATCH 034/667] Modify base class and add parsing process. --- .../resources/sql/server_azure_ad_administrators.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py b/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py index f68f91e9d..189a3cf2d 100644 --- a/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py +++ b/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py @@ -3,24 +3,23 @@ from azure.mgmt.sql import SqlManagementClient from msrestazure.azure_exceptions import CloudError -from ScoutSuite.providers.base.configs.resource_config import ResourceConfig +from ScoutSuite.providers.base.configs.resources import Resources -class ServerAzureAdAdministratorsConfig(ResourceConfig): +class ServerAzureAdAdministrators(Resources): def __init__(self, resource_group_name, server_name): self.resource_group_name = resource_group_name self.server_name = server_name - self.value = None - - async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): + async def fetch_all(self, credentials): # sdk container: api = SqlManagementClient(credentials.credentials, credentials.subscription_id) try: - self.value = \ + admins = \ api.server_azure_ad_administrators.get(self.resource_group_name, self.server_name) # TODO: await api.server_azure_ad_administrators.get(self.resource_group_name, self.server_name) + self['ad_admin_configured'] = True except CloudError: # no ad admin configured returns a 404 error - pass + self['ad_admin_configured'] = False From f7083a1e9eff628b0b4ec7a90ae072aa09c35fc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Wed, 20 Feb 2019 23:46:25 -0500 Subject: [PATCH 035/667] Modify base class and add parsing process. --- .../resources/sql/transparent_data_encryptions.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py b/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py index e2a582046..95a971bf1 100644 --- a/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py +++ b/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py @@ -2,10 +2,10 @@ from azure.mgmt.sql import SqlManagementClient -from ScoutSuite.providers.base.configs.resource_config import ResourceConfig +from ScoutSuite.providers.base.configs.resources import Resources -class TransparentDataEncryptionsConfig(ResourceConfig): +class TransparentDataEncryptions(Resources): def __init__(self, resource_group_name, server_name, database_name): self.resource_group_name = resource_group_name @@ -18,7 +18,12 @@ async def fetch_all(self, credentials, regions=None, partition_name='aws', targe # sdk container: api = SqlManagementClient(credentials.credentials, credentials.subscription_id) - self.value =\ + encryptions =\ api.transparent_data_encryptions.get(self.resource_group_name, self.server_name, self.database_name) # TODO: await api.transparent_data_encryptions.get(self.resource_group_name, self.server_name, self.database_name) + self['transparent_data_encryption_enabled'] = self._is_transparent_data_encryption_enabled(encryptions) + + @staticmethod + def _is_transparent_data_encryption_enabled(encryptions): + return encryptions.status == "Enabled" From 3f94312ba7b0dd3679f3ee4929246f238f94bd20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Wed, 20 Feb 2019 23:46:55 -0500 Subject: [PATCH 036/667] Modify base class and remove parsing processes of children. --- .../azure/resources/sql/databases.py | 63 +++++-------------- 1 file changed, 17 insertions(+), 46 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/databases.py b/ScoutSuite/providers/azure/resources/sql/databases.py index 732aabfdf..c58891f6e 100644 --- a/ScoutSuite/providers/azure/resources/sql/databases.py +++ b/ScoutSuite/providers/azure/resources/sql/databases.py @@ -2,68 +2,39 @@ from azure.mgmt.sql import SqlManagementClient -from ScoutSuite.providers.base.configs.resource_config import ResourceConfig +from ScoutSuite.providers.base.configs.resources import Resources -from .database_blob_auditing_policies import DatabaseBlobAuditingPoliciesConfig -from .database_threat_detection_policies import DatabaseThreatDetectionPoliciesConfig -from .replication_links import ReplicationLinksConfig -from .transparent_data_encryptions import TransparentDataEncryptionsConfig +from .database_blob_auditing_policies import DatabaseBlobAuditingPolicies +from .database_threat_detection_policies import DatabaseThreatDetectionPolicies +from .replication_links import ReplicationLinks +from .transparent_data_encryptions import TransparentDataEncryptions -class DatabasesConfig(ResourceConfig): - # the following static methods will be used to parse the config of each resource nested in a database config - # (defined in 'children' attribute below): - @staticmethod - def _parse_auditing_config(config): - return { - 'auditing_enabled': config.value.state == "Enabled" - } - - @staticmethod - def _parse_threat_detection_config(config): - return { - 'threat_detection_enabled': config.value.state == "Enabled" - } - - @staticmethod - def _parse_replication_links_config(config): - return { - 'replication_configured': len(config.value) > 0 - } - - @staticmethod - def _parse_transparent_data_encryption_config(config): - return { - 'transparent_data_encryption_enabled': config.value.status == "Enabled" - } - - # register children resources: +class Databases(Resources): children = [ - (DatabaseBlobAuditingPoliciesConfig, _parse_auditing_config), - (DatabaseThreatDetectionPoliciesConfig, _parse_threat_detection_config), - (ReplicationLinksConfig, _parse_replication_links_config), - (TransparentDataEncryptionsConfig, _parse_transparent_data_encryption_config) + DatabaseBlobAuditingPolicies, + DatabaseThreatDetectionPolicies, + ReplicationLinks, + TransparentDataEncryptions, ] def __init__(self, resource_group_name, server_name): self.resource_group_name = resource_group_name self.server_name = server_name - self.databases = {} - - async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): + async def fetch_all(self, credentials): # sdk container: api = SqlManagementClient(credentials.credentials, credentials.subscription_id) + self['databases'] = {} # TODO: for db in await api.databases.list_by_server(self.resource_group_name, self.server_name): for db in api.databases.list_by_server(self.resource_group_name, self.server_name): - self.databases[db.name] = { + self['databases'][db.name] = { 'id': db.name, } # put the following code in a fetch_children() parent method (typical method for a composite node)? - for (resource_config, resource_parser) in self.children: - resource = resource_config(self.resource_group_name, self.server_name, db.name) - await resource.fetch_all(credentials) - for k, v in resource_parser.__func__(resource).items(): - self.databases[db.name][k] = v + for resources_class in self.simple_children: + resources = resources_class(self.resource_group_name, self.server_name, db.name) + await resources.fetch_all(credentials) + self['databases'][db.name].update(resources) From 7ea7068ae8e34b9995ebb40faa125514f8ae9b0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Wed, 20 Feb 2019 23:47:07 -0500 Subject: [PATCH 037/667] Modify base class and remove parsing processes of children. --- .../providers/azure/resources/sql/servers.py | 48 ++++++------------- 1 file changed, 14 insertions(+), 34 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/servers.py b/ScoutSuite/providers/azure/resources/sql/servers.py index a44b913c9..48b5de91b 100644 --- a/ScoutSuite/providers/azure/resources/sql/servers.py +++ b/ScoutSuite/providers/azure/resources/sql/servers.py @@ -2,57 +2,37 @@ from azure.mgmt.sql import SqlManagementClient -from ScoutSuite.providers.base.configs.resource_config import ResourceConfig +from ScoutSuite.providers.base.configs.resources import Resources from ScoutSuite.providers.azure.utils import get_resource_group_name -from .databases import DatabasesConfig -from .server_azure_ad_administrators import ServerAzureAdAdministratorsConfig +from .databases import Databases +from .server_azure_ad_administrators import ServerAzureAdAdministrators from ..utils import get_non_provider_id -class ServersConfig(ResourceConfig): - # the following static methods will be used to parse the config of each resource nested in a server config - # (defined in 'children' attribute below): - @staticmethod - def _parse_databases_config(config): - return { - 'databases': config.databases - } - - @staticmethod - def _parse_azure_ad_administrators_config(config): - return { - 'ad_admin_configured': config.value is not None - } - - # register children resources: +class Servers(Resources): children = [ - (DatabasesConfig, _parse_databases_config), - (ServerAzureAdAdministratorsConfig, _parse_azure_ad_administrators_config) + Databases, + ServerAzureAdAdministrators, ] - def __init__(self): - self.servers = {} - - async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): + async def fetch_all(self, credentials): # sdk container: api = SqlManagementClient(credentials.credentials, credentials.subscription_id) - # for server in await api.servers.list(): + self['servers'] = {} + # TODO: for server in await api.servers.list(): for server in api.servers.list(): - # parsing: - id = get_non_provider_id(server.id) resource_group_name = get_resource_group_name(server.id) - self.servers[id] = { + self['servers'][id] = { 'id': id, 'name': server.name } # put the following code in a fetch_children() parent method (typical method for a composite node)? - for (resource_config, resource_parser) in self.children: - resource = resource_config(resource_group_name, server.name) - await resource.fetch_all(credentials) - for k, v in resource_parser.__func__(resource).items(): - self.servers[id][k] = v + for resources_class in self.children: + resources = resources_class(resource_group_name, server.name) + await resources.fetch_all(credentials) + self['servers'][id].update(resources) From 3f425d0e9be4d96964df767f1a321ef02aca3480 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Wed, 20 Feb 2019 23:53:09 -0500 Subject: [PATCH 038/667] Rename ResourceConfig class to Resources, make it abstract and make fetch_all method more general. --- ScoutSuite/providers/base/configs/resources.py | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 ScoutSuite/providers/base/configs/resources.py diff --git a/ScoutSuite/providers/base/configs/resources.py b/ScoutSuite/providers/base/configs/resources.py new file mode 100644 index 000000000..d709c4c3c --- /dev/null +++ b/ScoutSuite/providers/base/configs/resources.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- + +import abc + + +class Resources(dict, metaclass=abc.ABCMeta): + + @abc.abstractmethod + async def fetch_all(self, **kwargs): + raise NotImplementedError() From c3ba74e3a419a1de1a714bd657fb3dc487f73b86 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Thu, 21 Feb 2019 09:11:31 -0500 Subject: [PATCH 039/667] Added a bit of syntactic sugar to simplify the code --- ScoutSuite/providers/aws/services/awslambda.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/services/awslambda.py b/ScoutSuite/providers/aws/services/awslambda.py index 766c8bf0c..6ab9031a6 100644 --- a/ScoutSuite/providers/aws/services/awslambda.py +++ b/ScoutSuite/providers/aws/services/awslambda.py @@ -16,7 +16,7 @@ async def fetch_all(self, regions=None, partition_name='aws'): ) for region in self['regions']: - functions = LambdasConfig() + self['regions'][region] = await RegionalLambdas().fetch_all(region=region) await functions.fetch_all(region=region) self['regions'][region] = functions @@ -33,6 +33,7 @@ async def fetch_all(self, region): self['functions_count'] = len(functions) self['functions'] = functions + return self @staticmethod def parse_function(function): From b000ea463f60516307ea074902e00682ee13651b Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Thu, 21 Feb 2019 09:16:02 -0500 Subject: [PATCH 040/667] Renamed AwsFactory to AWSFactory --- ScoutSuite/providers/aws/aws_facade.py | 2 +- ScoutSuite/providers/aws/configs/regions_config.py | 4 ++-- ScoutSuite/providers/aws/services/awslambda.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ScoutSuite/providers/aws/aws_facade.py b/ScoutSuite/providers/aws/aws_facade.py index 19272c66a..4bdffc36b 100644 --- a/ScoutSuite/providers/aws/aws_facade.py +++ b/ScoutSuite/providers/aws/aws_facade.py @@ -3,7 +3,7 @@ from collections import Counter # TODO: Handle authentication better. I don't even know how it currently works. I think connect_service is called somewhere. -class AwsFacade(object): +class AWSFacade(object): def get_lambda_functions(self, region): aws_lambda = boto3.client('lambda', region_name=region) diff --git a/ScoutSuite/providers/aws/configs/regions_config.py b/ScoutSuite/providers/aws/configs/regions_config.py index ced0170f2..5082c4f18 100644 --- a/ScoutSuite/providers/aws/configs/regions_config.py +++ b/ScoutSuite/providers/aws/configs/regions_config.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- from ScoutSuite.providers.base.configs.resources import Resources -from ScoutSuite.providers.aws.aws_facade import AwsFacade +from ScoutSuite.providers.aws.aws_facade import AWSFacade class RegionsConfig(Resources): @@ -10,7 +10,7 @@ def __init__(self, service): async def fetch_all(self, chosen_regions=None, partition_name='aws'): # TODO: Should be injected - facade = AwsFacade() + facade = AWSFacade() self['regions'] = {} for region in await facade.build_region_list(self._service, chosen_regions, partition_name): diff --git a/ScoutSuite/providers/aws/services/awslambda.py b/ScoutSuite/providers/aws/services/awslambda.py index 6ab9031a6..991b1621c 100644 --- a/ScoutSuite/providers/aws/services/awslambda.py +++ b/ScoutSuite/providers/aws/services/awslambda.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from ScoutSuite.providers.base.configs.resources import Resources from ScoutSuite.providers.aws.configs.regions_config import RegionsConfig -from ScoutSuite.providers.aws.aws_facade import AwsFacade +from ScoutSuite.providers.aws.aws_facade import AWSFacade from opinel.utils.aws import build_region_list @@ -24,7 +24,7 @@ async def fetch_all(self, regions=None, partition_name='aws'): class LambdasConfig(Resources): async def fetch_all(self, region): # TODO: Should be injected - facade = AwsFacade() + facade = AWSFacade() functions = {} for raw_function in facade.get_lambda_functions(region): From f5295ab7a8427e2c5554c3d8e18d7059567ef36a Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Thu, 21 Feb 2019 09:16:50 -0500 Subject: [PATCH 041/667] Renamed LambdasConfig to RegionalLambdas --- ScoutSuite/providers/aws/services/awslambda.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/services/awslambda.py b/ScoutSuite/providers/aws/services/awslambda.py index 991b1621c..5f71fb943 100644 --- a/ScoutSuite/providers/aws/services/awslambda.py +++ b/ScoutSuite/providers/aws/services/awslambda.py @@ -21,7 +21,7 @@ async def fetch_all(self, regions=None, partition_name='aws'): self['regions'][region] = functions -class LambdasConfig(Resources): +class RegionalLambdas(Resources): async def fetch_all(self, region): # TODO: Should be injected facade = AWSFacade() From bac7a11e4507bd8d6e63be9dcbcea7433af76616 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Thu, 21 Feb 2019 09:17:11 -0500 Subject: [PATCH 042/667] Added parameter for compatibility, renamed LambdaServiceConfig to Lambdas --- ScoutSuite/providers/aws/configs/services.py | 4 ++-- ScoutSuite/providers/aws/services/awslambda.py | 11 +++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index 3c0c77084..f0c4e8373 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -2,7 +2,7 @@ from opinel.utils.console import printError, printException, printInfo, printDebug -from ScoutSuite.providers.aws.services.awslambda import LambdaServiceConfig +from ScoutSuite.providers.aws.services.awslambda import Lambdas from ScoutSuite.providers.aws.services.cloudformation import CloudFormationConfig from ScoutSuite.providers.aws.services.cloudtrail import CloudTrailConfig from ScoutSuite.providers.aws.services.cloudwatch import CloudWatchConfig @@ -70,7 +70,7 @@ def __init__(self, metadata=None, thread_config=4, **kwargs): self.elbv2 = ELBv2Config(metadata['compute']['elbv2'], thread_config) self.emr = EMRConfig(metadata['analytics']['emr'], thread_config) self.iam = IAMConfig(thread_config) - self.awslambda = LambdaServiceConfig() + self.awslambda = Lambdas() self.redshift = RedshiftConfig(metadata['database']['redshift'], thread_config) self.rds = RDSConfig(metadata['database']['rds'], thread_config) self.route53 = Route53Config(thread_config) diff --git a/ScoutSuite/providers/aws/services/awslambda.py b/ScoutSuite/providers/aws/services/awslambda.py index 5f71fb943..e7a6f10bb 100644 --- a/ScoutSuite/providers/aws/services/awslambda.py +++ b/ScoutSuite/providers/aws/services/awslambda.py @@ -5,20 +5,19 @@ from opinel.utils.aws import build_region_list -class LambdaServiceConfig(RegionsConfig): +class Lambdas(RegionsConfig): def __init__(self): - super(LambdaServiceConfig, self).__init__('lambda') + super(Lambdas, self).__init__('lambda') - async def fetch_all(self, regions=None, partition_name='aws'): - await super(LambdaServiceConfig, self).fetch_all( + # TODO: Remove the credentials parameter. We had to keep it for compatibility + async def fetch_all(self, credentials=None, regions=None, partition_name='aws'): + await super(Lambdas, self).fetch_all( chosen_regions=regions, partition_name=partition_name ) for region in self['regions']: self['regions'][region] = await RegionalLambdas().fetch_all(region=region) - await functions.fetch_all(region=region) - self['regions'][region] = functions class RegionalLambdas(Resources): From bbcbf8d14a09d68691570da4484290538ba28fc8 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Thu, 21 Feb 2019 09:25:36 -0500 Subject: [PATCH 043/667] Tried fixing tests. --- tests/test-main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test-main.py b/tests/test-main.py index b72a35218..cdeef76a3 100644 --- a/tests/test-main.py +++ b/tests/test-main.py @@ -33,14 +33,14 @@ def setUp(self): def tearDown(self): patch.stopall() - def test_empty(self): + async def test_empty(self): args = None code = None with patch("sys.stderr", return_value=MagicMock()): with self.assertRaises(SystemExit): args = ScoutSuiteArgumentParser().parse_args(args) - code = main(args) + code = await main(args) assert (code is None) From 522a6b99d4cd3e2d13be087c550bb509c978278c Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Thu, 21 Feb 2019 09:32:08 -0500 Subject: [PATCH 044/667] Tried making some test methods async --- tests/test-main.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/test-main.py b/tests/test-main.py index cdeef76a3..b748979f9 100644 --- a/tests/test-main.py +++ b/tests/test-main.py @@ -44,12 +44,12 @@ async def test_empty(self): assert (code is None) - def test_aws_provider(self): + async def test_aws_provider(self): args = ['aws'] self.mocked_provider.provider_code = "aws" args = ScoutSuiteArgumentParser().parse_args(args) - code = main(args) + code = await main(args) success_code = 0 assert (code == success_code) @@ -59,12 +59,12 @@ def test_aws_provider(self): assert (report_init_args[1] == "aws-default") # report_file_name assert (report_init_args[2] == "scoutsuite-report") # report_dir - def test_gcp_provider(self): + async def test_gcp_provider(self): args = ["gcp", "--service-account", "fakecredentials"] self.mocked_provider.provider_code = "gcp" args = ScoutSuiteArgumentParser().parse_args(args) - code = main(args) + code = await main(args) success_code = 0 assert (code == success_code) @@ -74,12 +74,12 @@ def test_gcp_provider(self): assert (report_init_args[1] == "gcp") # report_file_name assert (report_init_args[2] == "scoutsuite-report") # report_dir - def test_azure_provider(self): + async def test_azure_provider(self): args = ["azure", "--cli"] self.mocked_provider.provider_code = "azure" args = ScoutSuiteArgumentParser().parse_args(args) - code = main(args) + code = await main(args) success_code = 0 assert (code == success_code) @@ -89,18 +89,18 @@ def test_azure_provider(self): assert (report_init_args[1] == "azure") # report_file_name assert (report_init_args[2] == "scoutsuite-report") # report_dir - def test_unauthenticated(self): + async def test_unauthenticated(self): args = ["aws"] self.mocked_provider.provider_code = "aws" self.mocked_provider.authenticate = MagicMock(return_value=False) args = ScoutSuiteArgumentParser().parse_args(args) - code = main(args) + code = await main(args) unauthenticated_code = 42 assert (code == unauthenticated_code) - def test_keyboardinterrupted(self): + async def test_keyboardinterrupted(self): args = ["aws"] self.mocked_provider.provider_code = "aws" @@ -110,7 +110,7 @@ def _raise(e): self.mocked_provider.fetch = MagicMock(side_effect=_raise(KeyboardInterrupt)) args = ScoutSuiteArgumentParser().parse_args(args) - code = main(args) + code = await main(args) keyboardinterrupted_code = 130 assert (code == keyboardinterrupted_code) From 44217031560574a03e90b769d99ab4e6029743e6 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Thu, 21 Feb 2019 14:01:21 -0500 Subject: [PATCH 045/667] Started implementing EC2 and RegionalResources --- .../providers/aws/configs/regions_config.py | 29 +- ScoutSuite/providers/aws/configs/services.py | 4 +- ScoutSuite/providers/aws/provider.py | 12 +- ScoutSuite/providers/aws/services/ec2.py | 698 +++++++++--------- 4 files changed, 402 insertions(+), 341 deletions(-) diff --git a/ScoutSuite/providers/aws/configs/regions_config.py b/ScoutSuite/providers/aws/configs/regions_config.py index 5082c4f18..28d88619b 100644 --- a/ScoutSuite/providers/aws/configs/regions_config.py +++ b/ScoutSuite/providers/aws/configs/regions_config.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- from ScoutSuite.providers.base.configs.resources import Resources -from ScoutSuite.providers.aws.aws_facade import AWSFacade +from ScoutSuite.providers.aws.facade import AWSFacade +import abc class RegionsConfig(Resources): - def __init__(self, service): self._service = service @@ -15,5 +15,28 @@ async def fetch_all(self, chosen_regions=None, partition_name='aws'): self['regions'] = {} for region in await facade.build_region_list(self._service, chosen_regions, partition_name): self['regions'][region] = {} - + self['regions_count'] = len(self['regions']) + + +class RegionalResources(Resources): + def __init__(self, key): + self.key = key + + async def fetch_all(self, region): + resources = {} + for raw_resource in await self.get_regional_resources(region): + name, function = self.parse_resource(raw_resource) + resources[name] = function + + self[self.key + '_count'] = len(resources) + self[self.key] = resources + return self + + @abc.abstractclassmethod + def parse_resource(self, resource): + raise NotImplementedError() + + @abc.abstractclassmethod + async def get_regional_resources(self, region): + raise NotImplementedError() diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index 654ec6caa..8c3f7ed31 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -7,7 +7,7 @@ from ScoutSuite.providers.aws.services.cloudtrail import CloudTrailConfig from ScoutSuite.providers.aws.services.cloudwatch import CloudWatchConfig from ScoutSuite.providers.aws.services.directconnect import DirectConnectConfig -from ScoutSuite.providers.aws.services.ec2 import EC2Config +from ScoutSuite.providers.aws.services.ec2 import EC2 from ScoutSuite.providers.aws.services.efs import EFSConfig from ScoutSuite.providers.aws.services.elasticache import ElastiCacheConfig from ScoutSuite.providers.aws.services.elb import ELBConfig @@ -57,7 +57,7 @@ def __init__(self, metadata=None, thread_config=4, **kwargs): self.cloudtrail = CloudTrailConfig(metadata['management']['cloudtrail'], thread_config) self.cloudwatch = CloudWatchConfig(metadata['management']['cloudwatch'], thread_config) self.directconnect = DirectConnectConfig(metadata['network']['directconnect'], thread_config) - self.ec2 = EC2Config(metadata['compute']['ec2'], thread_config) + self.ec2 = EC2() self.efs = EFSConfig(metadata['storage']['efs'], thread_config) self.elasticache = ElastiCacheConfig(metadata['database']['elasticache'], thread_config) self.elb = ELBConfig(metadata['compute']['elb'], thread_config) diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index 4054370a4..f990fa13e 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -79,12 +79,12 @@ def preprocessing(self, ip_ranges=None, ip_ranges_name_key=None): # Various data processing calls self._check_ec2_zone_distribution() self._add_security_group_name_to_ec2_grants() - self._add_last_snapshot_date_to_ec2_volumes() - self._process_cloudtrail_trails(self.services['cloudtrail']) - self._add_cidr_display_name(ip_ranges, ip_ranges_name_key) - self._merge_route53_and_route53domains() - self._match_instances_and_roles() - self._match_iam_policies_and_buckets() + # self._add_last_snapshot_date_to_ec2_volumes() + # self._process_cloudtrail_trails(self.services['cloudtrail']) + # self._add_cidr_display_name(ip_ranges, ip_ranges_name_key) + # self._merge_route53_and_route53domains() + # self._match_instances_and_roles() + # self._match_iam_policies_and_buckets() super(AWSProvider, self).preprocessing() diff --git a/ScoutSuite/providers/aws/services/ec2.py b/ScoutSuite/providers/aws/services/ec2.py index 915589599..d454f544a 100644 --- a/ScoutSuite/providers/aws/services/ec2.py +++ b/ScoutSuite/providers/aws/services/ec2.py @@ -1,330 +1,368 @@ -# -*- coding: utf-8 -*- -""" -EC2-related classes and functions -""" - -# TODO: move a lot of this to VPCconfig, and use some sort of filter to only list SGs in EC2 classic -import netaddr -import base64 - -from opinel.utils.aws import get_name -from opinel.utils.console import printException, printInfo -from opinel.utils.fs import load_data -from opinel.utils.globals import manage_dictionary - -from ScoutSuite.providers.aws.configs.vpc import VPCConfig -from ScoutSuite.providers.base.configs.browser import get_attribute_at -from ScoutSuite.providers.base.provider import BaseProvider -from ScoutSuite.utils import get_keys, ec2_classic -from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients - - -######################################## -# Globals -######################################## - -icmp_message_types_dict = load_data('icmp_message_types.json', 'icmp_message_types') -protocols_dict = load_data('protocols.json', 'protocols') - - -######################################## -# EC2RegionConfig -######################################## - -class EC2RegionConfig(RegionConfig): - """ - EC2 configuration for a single AWS region - """ - - def parse_elastic_ip(self, global_params, region, eip): - """ - - :param global_params: - :param region: - :param eip: - :return: - """ - self.elastic_ips[eip['PublicIp']] = eip - - def parse_instance(self, global_params, region, reservation): - """ - Parse a single EC2 instance - - :param global_params: Parameters shared for all regions - :param region: Name of the AWS region - :param instance: Cluster - """ - for i in reservation['Instances']: - instance = {} - vpc_id = i['VpcId'] if 'VpcId' in i and i['VpcId'] else ec2_classic - manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types)) - instance['reservation_id'] = reservation['ReservationId'] - instance['id'] = i['InstanceId'] - instance['monitoring_enabled'] = i['Monitoring']['State'] == 'enabled' - instance['user_data'] = self._get_user_data(region, instance['id']) - get_name(i, instance, 'InstanceId') - get_keys(i, instance, ['KeyName', 'LaunchTime', 'InstanceType', 'State', 'IamInstanceProfile', 'SubnetId']) - # Network interfaces & security groups - manage_dictionary(instance, 'network_interfaces', {}) - for eni in i['NetworkInterfaces']: - nic = {} - get_keys(eni, nic, ['Association', 'Groups', 'PrivateIpAddresses', 'SubnetId', 'Ipv6Addresses']) - instance['network_interfaces'][eni['NetworkInterfaceId']] = nic - self.vpcs[vpc_id].instances[i['InstanceId']] = instance - - def _get_user_data(self, region, instance_id): - user_data_response = api_clients[region].describe_instance_attribute(Attribute='userData', InstanceId=instance_id) - - if 'Value' not in user_data_response['UserData'].keys(): - return None - - return base64.b64decode(user_data_response['UserData']['Value']).decode('utf-8') - - def parse_image(self, global_params, region, image): - """ - Parses a single AMI (Amazon Machine Image) - - :param global_params: Parameters shared for all regions - :param region: Name of the AWS region - :param snapshot: Single image - """ - id = image['ImageId'] - name = image['Name'] - - image['id'] = id - image['name'] = name - - self.images[id] = image - - def parse_security_group(self, global_params, region, group): - """ - Parse a single Redsfhit security group - - :param global_params: Parameters shared for all regions - :param region: Name of the AWS region - :param security)_group: Security group - """ - vpc_id = group['VpcId'] if 'VpcId' in group and group['VpcId'] else ec2_classic - manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types)) - security_group = {} - security_group['name'] = group['GroupName'] - security_group['id'] = group['GroupId'] - security_group['description'] = group['Description'] - security_group['owner_id'] = group['OwnerId'] - security_group['rules'] = {'ingress': {}, 'egress': {}} - security_group['rules']['ingress']['protocols'], security_group['rules']['ingress'][ - 'count'] = self.__parse_security_group_rules(group['IpPermissions']) - security_group['rules']['egress']['protocols'], security_group['rules']['egress'][ - 'count'] = self.__parse_security_group_rules(group['IpPermissionsEgress']) - self.vpcs[vpc_id].security_groups[group['GroupId']] = security_group - - def __parse_security_group_rules(self, rules): - """ - - :param self: - :param rules: - :return: - """ - protocols = {} - rules_count = 0 - for rule in rules: - ip_protocol = rule['IpProtocol'].upper() - if ip_protocol == '-1': - ip_protocol = 'ALL' - protocols = manage_dictionary(protocols, ip_protocol, {}) - protocols[ip_protocol] = manage_dictionary(protocols[ip_protocol], 'ports', {}) - # Save the port (single port or range) - port_value = 'N/A' - if 'FromPort' in rule and 'ToPort' in rule: - if ip_protocol == 'ICMP': - # FromPort with ICMP is the type of message - port_value = icmp_message_types_dict[str(rule['FromPort'])] - elif rule['FromPort'] == rule['ToPort']: - port_value = str(rule['FromPort']) - else: - port_value = '%s-%s' % (rule['FromPort'], rule['ToPort']) - manage_dictionary(protocols[ip_protocol]['ports'], port_value, {}) - # Save grants, values are either a CIDR or an EC2 security group - for grant in rule['UserIdGroupPairs']: - manage_dictionary(protocols[ip_protocol]['ports'][port_value], 'security_groups', []) - protocols[ip_protocol]['ports'][port_value]['security_groups'].append(grant) - rules_count = rules_count + 1 - for grant in rule['IpRanges']: - manage_dictionary(protocols[ip_protocol]['ports'][port_value], 'cidrs', []) - protocols[ip_protocol]['ports'][port_value]['cidrs'].append({'CIDR': grant['CidrIp']}) - rules_count = rules_count + 1 - # IPv6 - for grant in rule['Ipv6Ranges']: - manage_dictionary(protocols[ip_protocol]['ports'][port_value], 'cidrs', []) - protocols[ip_protocol]['ports'][port_value]['cidrs'].append({'CIDR': grant['CidrIpv6']}) - rules_count = rules_count + 1 - - return protocols, rules_count - - def parse_snapshot(self, global_params, region, snapshot): - """ - - :param global_params: Parameters shared for all regions - :param region: Name of the AWS region - :param snapshot: Single snapshot - :return: - """ - snapshot['id'] = snapshot.pop('SnapshotId') - snapshot['name'] = get_name(snapshot, snapshot, 'id') - self.snapshots[snapshot['id']] = snapshot - # Get snapshot attribute - snapshot['createVolumePermission'] = \ - api_clients[region].describe_snapshot_attribute(Attribute='createVolumePermission', SnapshotId=snapshot['id'])[ - 'CreateVolumePermissions'] - snapshot['public'] = self._is_public(snapshot) - - def _is_public(self, snapshot): - return any([permission.get('Group') == 'all' for permission in snapshot['createVolumePermission']]) - - def parse_volume(self, global_params, region, volume): - """ - - :param global_params: Parameters shared for all regions - :param region: Name of the AWS region - :param volume: Single EBS volume - :return: - """ - volume['id'] = volume.pop('VolumeId') - volume['name'] = get_name(volume, volume, 'id') - self.volumes[volume['id']] = volume - - -######################################## -# EC2Config -######################################## - -class EC2Config(RegionalServiceConfig): - """ - EC2 configuration for all AWS regions - """ - - region_config_class = EC2RegionConfig - - def __init__(self, service_metadata, thread_config=4): - super(EC2Config, self).__init__(service_metadata, thread_config) - - -######################################## -##### EC2 analysis functions -######################################## - -def analyze_ec2_config(ec2_info, aws_account_id, force_write): - try: - printInfo('Analyzing EC2 config... ', newLine=False) - # Custom EC2 analysis - # check_for_elastic_ip(ec2_info) - # FIXME - commented for now as this method doesn't seem to be defined anywhere' - # list_network_attack_surface(ec2_info, 'attack_surface', 'PublicIpAddress') - # TODO: make this optional, commented out for now - # list_network_attack_surface(ec2_info, 'private_attack_surface', 'PrivateIpAddress') - printInfo('Success') - except Exception as e: - printInfo('Error') - printException(e) - -def add_security_group_name_to_ec2_grants_callback(ec2_config, current_config, path, current_path, ec2_grant, - callback_args): - """ - Callback - - :param ec2_config: - :param current_config: - :param path: - :param current_path: - :param ec2_grant: - :param callback_args: - :return: - """ - sg_id = ec2_grant['GroupId'] - if sg_id in current_path: - target = current_path[:(current_path.index(sg_id) + 1)] - ec2_grant['GroupName'] = get_attribute_at(ec2_config, target, 'name') - elif ec2_grant['UserId'] == callback_args['AWSAccountId']: - if 'VpcId' in ec2_grant: - target = current_path[:(current_path.index('vpcs') + 1)] - target.append(ec2_grant['VpcId']) - target.append('security_groups') - target.append(sg_id) - else: - target = current_path[:(current_path.index('security_groups') + 1)] - target.append(sg_id) - ec2_grant['GroupName'] = get_attribute_at(ec2_config, target, 'name') - - -def check_for_elastic_ip(ec2_info): - """ - Check that the whitelisted EC2 IP addresses are not static IPs - - :param ec2_info: - :return: - """ - # Build a list of all elatic IP in the account - elastic_ips = [] - for region in ec2_info['regions']: - if 'elastic_ips' in ec2_info['regions'][region]: - for eip in ec2_info['regions'][region]['elastic_ips']: - elastic_ips.append(eip) - new_items = [] - new_macro_items = [] - for i, item in enumerate(ec2_info['violations']['non-elastic-ec2-public-ip-whitelisted'].items): - ip = netaddr.IPNetwork(item) - found = False - for eip in elastic_ips: - eip = netaddr.IPNetwork(eip) - if ip in eip: - found = True - break - if not found: - new_items.append(ec2_info['violations']['non-elastic-ec2-public-ip-whitelisted'].items[i]) - new_macro_items.append(ec2_info['violations']['non-elastic-ec2-public-ip-whitelisted'].macro_items[i]) - ec2_info['violations']['non-elastic-ec2-public-ip-whitelisted'].items = new_items - ec2_info['violations']['non-elastic-ec2-public-ip-whitelisted'].macro_items = new_macro_items - - -def link_elastic_ips_callback2(ec2_config, current_config, path, current_path, instance_id, callback_args): - if instance_id == callback_args['instance_id']: - if not 'PublicIpAddress' in current_config: - current_config['PublicIpAddress'] = callback_args['elastic_ip'] - elif current_config['PublicIpAddress'] != callback_args['elastic_ip']: - printInfo('Warning: public IP address exists (%s) for an instance associated with an elastic IP (%s)' % ( - current_config['PublicIpAddress'], callback_args['elastic_ip'])) - # This can happen... fix it - - -def list_instances_in_security_groups(region_info): - """ - Once all the data has been fetched, iterate through instances and list them - Could this be done when all the "used_by" values are set ??? TODO - - :param region_info: - :return: - """ - for vpc in region_info['vpcs']: - if not 'instances' in region_info['vpcs'][vpc]: - return - for instance in region_info['vpcs'][vpc]['instances']: - state = region_info['vpcs'][vpc]['instances'][instance]['State']['Name'] - for sg in region_info['vpcs'][vpc]['instances'][instance]['security_groups']: - sg_id = sg['GroupId'] - manage_dictionary(region_info['vpcs'][vpc]['security_groups'][sg_id], 'instances', {}) - manage_dictionary(region_info['vpcs'][vpc]['security_groups'][sg_id]['instances'], state, []) - region_info['vpcs'][vpc]['security_groups'][sg_id]['instances'][state].append(instance) - - -def manage_vpc(vpc_info, vpc_id): - """ - Ensure name and ID are set - - :param vpc_info: - :param vpc_id: - :return: - """ - manage_dictionary(vpc_info, vpc_id, {}) - vpc_info[vpc_id]['id'] = vpc_id - if not 'name' in vpc_info[vpc_id]: - vpc_info[vpc_id]['name'] = vpc_id +from ScoutSuite.providers.base.configs.resources import Resources +from ScoutSuite.providers.aws.configs.regions_config import RegionsConfig, RegionalResources +from ScoutSuite.providers.aws.facade import AWSFacade + + +class EC2(RegionsConfig): + def __init__(self): + super(EC2, self).__init__('ec2') + + # TODO: Remove the credentials parameter. We had to keep it for compatibility + async def fetch_all(self, credentials=None, regions=None, partition_name='aws'): + await super(EC2, self).fetch_all(chosen_regions=regions, partition_name=partition_name) + + for region in self['regions']: + self['regions'][region] = await EC2Instances().fetch_all(region) + + +class EC2Instances(RegionalResources): + def __init__(self): + self.facade = AWSFacade() + super(EC2Instances, self).__init__('instances') + + def parse_resource(self, resource): + return resource['name'], resource + + async def get_regional_resources(self, region): + return [ + { + 'name': 'lel', + 'some attribute': True + }, + { + 'name': 'lel2', + 'some attribute': True + }, + ] + + +# # -*- coding: utf-8 -*- +# """ +# EC2-related classes and functions +# """ + +# # TODO: move a lot of this to VPCconfig, and use some sort of filter to only list SGs in EC2 classic +# import netaddr +# import base64 + +# from opinel.utils.aws import get_name +# from opinel.utils.console import printException, printInfo +# from opinel.utils.fs import load_data +# from opinel.utils.globals import manage_dictionary + +# from ScoutSuite.providers.aws.configs.vpc import VPCConfig +# from ScoutSuite.providers.base.configs.browser import get_attribute_at +# from ScoutSuite.providers.base.provider import BaseProvider +# from ScoutSuite.utils import get_keys, ec2_classic +# from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients + + +# ######################################## +# # Globals +# ######################################## + +# icmp_message_types_dict = load_data('icmp_message_types.json', 'icmp_message_types') +# protocols_dict = load_data('protocols.json', 'protocols') + + +# ######################################## +# # EC2RegionConfig +# ######################################## + +# class EC2RegionConfig(RegionConfig): +# """ +# EC2 configuration for a single AWS region +# """ + +# def parse_elastic_ip(self, global_params, region, eip): +# """ + +# :param global_params: +# :param region: +# :param eip: +# :return: +# """ +# self.elastic_ips[eip['PublicIp']] = eip + +# def parse_instance(self, global_params, region, reservation): +# """ +# Parse a single EC2 instance + +# :param global_params: Parameters shared for all regions +# :param region: Name of the AWS region +# :param instance: Cluster +# """ +# for i in reservation['Instances']: +# instance = {} +# vpc_id = i['VpcId'] if 'VpcId' in i and i['VpcId'] else ec2_classic +# manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types)) +# instance['reservation_id'] = reservation['ReservationId'] +# instance['id'] = i['InstanceId'] +# instance['monitoring_enabled'] = i['Monitoring']['State'] == 'enabled' +# instance['user_data'] = self._get_user_data(region, instance['id']) +# get_name(i, instance, 'InstanceId') +# get_keys(i, instance, ['KeyName', 'LaunchTime', 'InstanceType', 'State', 'IamInstanceProfile', 'SubnetId']) +# # Network interfaces & security groups +# manage_dictionary(instance, 'network_interfaces', {}) +# for eni in i['NetworkInterfaces']: +# nic = {} +# get_keys(eni, nic, ['Association', 'Groups', 'PrivateIpAddresses', 'SubnetId', 'Ipv6Addresses']) +# instance['network_interfaces'][eni['NetworkInterfaceId']] = nic +# self.vpcs[vpc_id].instances[i['InstanceId']] = instance + +# def _get_user_data(self, region, instance_id): +# user_data_response = api_clients[region].describe_instance_attribute(Attribute='userData', InstanceId=instance_id) + +# if 'Value' not in user_data_response['UserData'].keys(): +# return None + +# return base64.b64decode(user_data_response['UserData']['Value']).decode('utf-8') + +# def parse_image(self, global_params, region, image): +# """ +# Parses a single AMI (Amazon Machine Image) + +# :param global_params: Parameters shared for all regions +# :param region: Name of the AWS region +# :param snapshot: Single image +# """ +# id = image['ImageId'] +# name = image['Name'] + +# image['id'] = id +# image['name'] = name + +# self.images[id] = image + +# def parse_security_group(self, global_params, region, group): +# """ +# Parse a single Redsfhit security group + +# :param global_params: Parameters shared for all regions +# :param region: Name of the AWS region +# :param security)_group: Security group +# """ +# vpc_id = group['VpcId'] if 'VpcId' in group and group['VpcId'] else ec2_classic +# manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types)) +# security_group = {} +# security_group['name'] = group['GroupName'] +# security_group['id'] = group['GroupId'] +# security_group['description'] = group['Description'] +# security_group['owner_id'] = group['OwnerId'] +# security_group['rules'] = {'ingress': {}, 'egress': {}} +# security_group['rules']['ingress']['protocols'], security_group['rules']['ingress'][ +# 'count'] = self.__parse_security_group_rules(group['IpPermissions']) +# security_group['rules']['egress']['protocols'], security_group['rules']['egress'][ +# 'count'] = self.__parse_security_group_rules(group['IpPermissionsEgress']) +# self.vpcs[vpc_id].security_groups[group['GroupId']] = security_group + +# def __parse_security_group_rules(self, rules): +# """ + +# :param self: +# :param rules: +# :return: +# """ +# protocols = {} +# rules_count = 0 +# for rule in rules: +# ip_protocol = rule['IpProtocol'].upper() +# if ip_protocol == '-1': +# ip_protocol = 'ALL' +# protocols = manage_dictionary(protocols, ip_protocol, {}) +# protocols[ip_protocol] = manage_dictionary(protocols[ip_protocol], 'ports', {}) +# # Save the port (single port or range) +# port_value = 'N/A' +# if 'FromPort' in rule and 'ToPort' in rule: +# if ip_protocol == 'ICMP': +# # FromPort with ICMP is the type of message +# port_value = icmp_message_types_dict[str(rule['FromPort'])] +# elif rule['FromPort'] == rule['ToPort']: +# port_value = str(rule['FromPort']) +# else: +# port_value = '%s-%s' % (rule['FromPort'], rule['ToPort']) +# manage_dictionary(protocols[ip_protocol]['ports'], port_value, {}) +# # Save grants, values are either a CIDR or an EC2 security group +# for grant in rule['UserIdGroupPairs']: +# manage_dictionary(protocols[ip_protocol]['ports'][port_value], 'security_groups', []) +# protocols[ip_protocol]['ports'][port_value]['security_groups'].append(grant) +# rules_count = rules_count + 1 +# for grant in rule['IpRanges']: +# manage_dictionary(protocols[ip_protocol]['ports'][port_value], 'cidrs', []) +# protocols[ip_protocol]['ports'][port_value]['cidrs'].append({'CIDR': grant['CidrIp']}) +# rules_count = rules_count + 1 +# # IPv6 +# for grant in rule['Ipv6Ranges']: +# manage_dictionary(protocols[ip_protocol]['ports'][port_value], 'cidrs', []) +# protocols[ip_protocol]['ports'][port_value]['cidrs'].append({'CIDR': grant['CidrIpv6']}) +# rules_count = rules_count + 1 + +# return protocols, rules_count + +# def parse_snapshot(self, global_params, region, snapshot): +# """ + +# :param global_params: Parameters shared for all regions +# :param region: Name of the AWS region +# :param snapshot: Single snapshot +# :return: +# """ +# snapshot['id'] = snapshot.pop('SnapshotId') +# snapshot['name'] = get_name(snapshot, snapshot, 'id') +# self.snapshots[snapshot['id']] = snapshot +# # Get snapshot attribute +# snapshot['createVolumePermission'] = \ +# api_clients[region].describe_snapshot_attribute(Attribute='createVolumePermission', SnapshotId=snapshot['id'])[ +# 'CreateVolumePermissions'] +# snapshot['public'] = self._is_public(snapshot) + +# def _is_public(self, snapshot): +# return any([permission.get('Group') == 'all' for permission in snapshot['createVolumePermission']]) + +# def parse_volume(self, global_params, region, volume): +# """ + +# :param global_params: Parameters shared for all regions +# :param region: Name of the AWS region +# :param volume: Single EBS volume +# :return: +# """ +# volume['id'] = volume.pop('VolumeId') +# volume['name'] = get_name(volume, volume, 'id') +# self.volumes[volume['id']] = volume + + +# ######################################## +# # EC2Config +# ######################################## + +# class EC2Config(RegionalServiceConfig): +# """ +# EC2 configuration for all AWS regions +# """ + +# region_config_class = EC2RegionConfig + +# def __init__(self, service_metadata, thread_config=4): +# super(EC2Config, self).__init__(service_metadata, thread_config) + + +# ######################################## +# ##### EC2 analysis functions +# ######################################## + +# def analyze_ec2_config(ec2_info, aws_account_id, force_write): +# try: +# printInfo('Analyzing EC2 config... ', newLine=False) +# # Custom EC2 analysis +# # check_for_elastic_ip(ec2_info) +# # FIXME - commented for now as this method doesn't seem to be defined anywhere' +# # list_network_attack_surface(ec2_info, 'attack_surface', 'PublicIpAddress') +# # TODO: make this optional, commented out for now +# # list_network_attack_surface(ec2_info, 'private_attack_surface', 'PrivateIpAddress') +# printInfo('Success') +# except Exception as e: +# printInfo('Error') +# printException(e) + +# def add_security_group_name_to_ec2_grants_callback(ec2_config, current_config, path, current_path, ec2_grant, +# callback_args): +# """ +# Callback + +# :param ec2_config: +# :param current_config: +# :param path: +# :param current_path: +# :param ec2_grant: +# :param callback_args: +# :return: +# """ +# sg_id = ec2_grant['GroupId'] +# if sg_id in current_path: +# target = current_path[:(current_path.index(sg_id) + 1)] +# ec2_grant['GroupName'] = get_attribute_at(ec2_config, target, 'name') +# elif ec2_grant['UserId'] == callback_args['AWSAccountId']: +# if 'VpcId' in ec2_grant: +# target = current_path[:(current_path.index('vpcs') + 1)] +# target.append(ec2_grant['VpcId']) +# target.append('security_groups') +# target.append(sg_id) +# else: +# target = current_path[:(current_path.index('security_groups') + 1)] +# target.append(sg_id) +# ec2_grant['GroupName'] = get_attribute_at(ec2_config, target, 'name') + + +# def check_for_elastic_ip(ec2_info): +# """ +# Check that the whitelisted EC2 IP addresses are not static IPs + +# :param ec2_info: +# :return: +# """ +# # Build a list of all elatic IP in the account +# elastic_ips = [] +# for region in ec2_info['regions']: +# if 'elastic_ips' in ec2_info['regions'][region]: +# for eip in ec2_info['regions'][region]['elastic_ips']: +# elastic_ips.append(eip) +# new_items = [] +# new_macro_items = [] +# for i, item in enumerate(ec2_info['violations']['non-elastic-ec2-public-ip-whitelisted'].items): +# ip = netaddr.IPNetwork(item) +# found = False +# for eip in elastic_ips: +# eip = netaddr.IPNetwork(eip) +# if ip in eip: +# found = True +# break +# if not found: +# new_items.append(ec2_info['violations']['non-elastic-ec2-public-ip-whitelisted'].items[i]) +# new_macro_items.append(ec2_info['violations']['non-elastic-ec2-public-ip-whitelisted'].macro_items[i]) +# ec2_info['violations']['non-elastic-ec2-public-ip-whitelisted'].items = new_items +# ec2_info['violations']['non-elastic-ec2-public-ip-whitelisted'].macro_items = new_macro_items + + +# def link_elastic_ips_callback2(ec2_config, current_config, path, current_path, instance_id, callback_args): +# if instance_id == callback_args['instance_id']: +# if not 'PublicIpAddress' in current_config: +# current_config['PublicIpAddress'] = callback_args['elastic_ip'] +# elif current_config['PublicIpAddress'] != callback_args['elastic_ip']: +# printInfo('Warning: public IP address exists (%s) for an instance associated with an elastic IP (%s)' % ( +# current_config['PublicIpAddress'], callback_args['elastic_ip'])) +# # This can happen... fix it + + +# def list_instances_in_security_groups(region_info): +# """ +# Once all the data has been fetched, iterate through instances and list them +# Could this be done when all the "used_by" values are set ??? TODO + +# :param region_info: +# :return: +# """ +# for vpc in region_info['vpcs']: +# if not 'instances' in region_info['vpcs'][vpc]: +# return +# for instance in region_info['vpcs'][vpc]['instances']: +# state = region_info['vpcs'][vpc]['instances'][instance]['State']['Name'] +# for sg in region_info['vpcs'][vpc]['instances'][instance]['security_groups']: +# sg_id = sg['GroupId'] +# manage_dictionary(region_info['vpcs'][vpc]['security_groups'][sg_id], 'instances', {}) +# manage_dictionary(region_info['vpcs'][vpc]['security_groups'][sg_id]['instances'], state, []) +# region_info['vpcs'][vpc]['security_groups'][sg_id]['instances'][state].append(instance) + + +# def manage_vpc(vpc_info, vpc_id): +# """ +# Ensure name and ID are set + +# :param vpc_info: +# :param vpc_id: +# :return: +# """ +# manage_dictionary(vpc_info, vpc_id, {}) +# vpc_info[vpc_id]['id'] = vpc_id +# if not 'name' in vpc_info[vpc_id]: +# vpc_info[vpc_id]['name'] = vpc_id From 85a2166e34f93d64cd17d928a32d48136601c06a Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Fri, 22 Feb 2019 09:54:37 -0500 Subject: [PATCH 046/667] Renamed aws_facade to facade --- ScoutSuite/providers/aws/aws_facade.py | 34 --------- ScoutSuite/providers/aws/facade.py | 71 +++++++++++++++++++ .../providers/aws/services/awslambda.py | 2 +- 3 files changed, 72 insertions(+), 35 deletions(-) delete mode 100644 ScoutSuite/providers/aws/aws_facade.py create mode 100644 ScoutSuite/providers/aws/facade.py diff --git a/ScoutSuite/providers/aws/aws_facade.py b/ScoutSuite/providers/aws/aws_facade.py deleted file mode 100644 index 4bdffc36b..000000000 --- a/ScoutSuite/providers/aws/aws_facade.py +++ /dev/null @@ -1,34 +0,0 @@ -import boto3 -from botocore.session import Session -from collections import Counter - -# TODO: Handle authentication better. I don't even know how it currently works. I think connect_service is called somewhere. -class AWSFacade(object): - def get_lambda_functions(self, region): - aws_lambda = boto3.client('lambda', region_name=region) - - functions = [] - - while True: - response = aws_lambda.list_functions() - - functions.extend(response['Functions']) - marker = response.get('NextMarker', None) - if marker is None: - break - - return functions - - async def build_region_list(self, service, chosen_regions=None, partition_name='aws'): - service = 'ec2containerservice' if service == 'ecs' else service - available_services = Session().get_available_services() - - if service not in available_services: - raise Exception('Service ' + service + ' is not available.') - - regions = Session().get_available_regions(service, partition_name) - - if chosen_regions: - return list((Counter(regions) & Counter(chosen_regions)).elements()) - else: - return regions diff --git a/ScoutSuite/providers/aws/facade.py b/ScoutSuite/providers/aws/facade.py new file mode 100644 index 000000000..75474a336 --- /dev/null +++ b/ScoutSuite/providers/aws/facade.py @@ -0,0 +1,71 @@ +import boto3 +from botocore.session import Session +from collections import Counter +import itertools +import base64 + +# TODO: It would be interesting to split the facade in different sub-facades. For example, a call could look like this: +# facade.ec2.get_instances(region) or +# facade.lambda.get_functions(region) + + +class AWSFacade(object): + async def build_region_list(self, service, chosen_regions=None, partition_name='aws'): + service = 'ec2containerservice' if service == 'ecs' else service + available_services = Session().get_available_services() + + if service not in available_services: + raise Exception('Service ' + service + ' is not available.') + + regions = Session().get_available_regions(service, partition_name) + + if chosen_regions: + return list((Counter(regions) & Counter(chosen_regions)).elements()) + else: + return regions + + def get_lambda_functions(self, region): + aws_lambda = boto3.client('lambda', region_name=region) + return self._get_all_pages( + lambda: aws_lambda.list_functions(), + lambda response: response['Functions'] + ) + + def get_ec2_instances(self, region, vpc): + ec2_client = boto3.client('ec2', region_name=region) + + return self._get_all_pages( + lambda: ec2_client.describe_instances( + Filters=[{'Name': 'vpc-id', 'Values': [vpc]}]), + lambda response: itertools.chain.from_iterable( + [reservation['Instances'] for reservation in response['Reservations']]) + ) + + def get_vpcs(self, region): + vpc_client = boto3.client('ec2', region_name=region) + return self._get_all_pages( + lambda: vpc_client.describe_vpcs(), + lambda response: response['Vpcs'] + ) + + def _get_all_pages(self, api_call, parse_response): + resources = [] + + while True: + response = api_call() + + resources.extend(parse_response(response)) + marker = response.get('NextMarker', None) + if marker is None: + break + return resources + + def get_ec2_instance_user_data(self, region, instance_id): + ec2_client = boto3.client('ec2', region_name=region) + user_data_response = ec2_client.describe_instance_attribute( + Attribute='userData', InstanceId=instance_id) + + if 'Value' not in user_data_response['UserData'].keys(): + return None + + return base64.b64decode(user_data_response['UserData']['Value']).decode('utf-8') diff --git a/ScoutSuite/providers/aws/services/awslambda.py b/ScoutSuite/providers/aws/services/awslambda.py index e7a6f10bb..b3c4d6fd4 100644 --- a/ScoutSuite/providers/aws/services/awslambda.py +++ b/ScoutSuite/providers/aws/services/awslambda.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from ScoutSuite.providers.base.configs.resources import Resources from ScoutSuite.providers.aws.configs.regions_config import RegionsConfig -from ScoutSuite.providers.aws.aws_facade import AWSFacade +from ScoutSuite.providers.aws.facade import AWSFacade from opinel.utils.aws import build_region_list From 67da28f30f379124befd35eb3972f2df5b9e8197 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Fri, 22 Feb 2019 09:56:13 -0500 Subject: [PATCH 047/667] Migrated ec2 instances and renamed RegionalResources to ScopedResources --- .../providers/aws/configs/regions_config.py | 12 +-- ScoutSuite/providers/aws/services/ec2.py | 81 ++++++++++++++----- 2 files changed, 69 insertions(+), 24 deletions(-) diff --git a/ScoutSuite/providers/aws/configs/regions_config.py b/ScoutSuite/providers/aws/configs/regions_config.py index 28d88619b..079e6fe2e 100644 --- a/ScoutSuite/providers/aws/configs/regions_config.py +++ b/ScoutSuite/providers/aws/configs/regions_config.py @@ -19,15 +19,15 @@ async def fetch_all(self, chosen_regions=None, partition_name='aws'): self['regions_count'] = len(self['regions']) -class RegionalResources(Resources): +class ScopedResources(Resources): def __init__(self, key): self.key = key - async def fetch_all(self, region): + async def fetch_all(self, scope): resources = {} - for raw_resource in await self.get_regional_resources(region): - name, function = self.parse_resource(raw_resource) - resources[name] = function + for raw_resource in await self.get_resources_in_scope(scope): + name, ressource = self.parse_resource(raw_resource) + resources[name] = ressource self[self.key + '_count'] = len(resources) self[self.key] = resources @@ -38,5 +38,5 @@ def parse_resource(self, resource): raise NotImplementedError() @abc.abstractclassmethod - async def get_regional_resources(self, region): + async def get_resources_in_scope(self, region): raise NotImplementedError() diff --git a/ScoutSuite/providers/aws/services/ec2.py b/ScoutSuite/providers/aws/services/ec2.py index d454f544a..387b63697 100644 --- a/ScoutSuite/providers/aws/services/ec2.py +++ b/ScoutSuite/providers/aws/services/ec2.py @@ -1,7 +1,10 @@ from ScoutSuite.providers.base.configs.resources import Resources -from ScoutSuite.providers.aws.configs.regions_config import RegionsConfig, RegionalResources +from ScoutSuite.providers.aws.configs.regions_config import RegionsConfig, ScopedResources from ScoutSuite.providers.aws.facade import AWSFacade +from opinel.utils.aws import get_name +from ScoutSuite.utils import get_keys, ec2_classic + class EC2(RegionsConfig): def __init__(self): @@ -12,29 +15,71 @@ async def fetch_all(self, credentials=None, regions=None, partition_name='aws'): await super(EC2, self).fetch_all(chosen_regions=regions, partition_name=partition_name) for region in self['regions']: - self['regions'][region] = await EC2Instances().fetch_all(region) + self['regions'][region] = await Vpcs().fetch_all(region) -class EC2Instances(RegionalResources): +class Vpcs(ScopedResources): def __init__(self): self.facade = AWSFacade() - super(EC2Instances, self).__init__('instances') + super(Vpcs, self).__init__('vpcs') + + async def fetch_all(self, region): + await super(Vpcs, self).fetch_all(region) + + for vpc in self['vpcs']: + # TODO: Add vpc_resource_types + instances = await EC2Instances(region).fetch_all(vpc) + self['vpcs'][vpc].update(instances) + + self['instances_count'] = sum([vpc['instances_count'] for vpc in self['vpcs'].values()]) + + return self - def parse_resource(self, resource): - return resource['name'], resource - - async def get_regional_resources(self, region): - return [ - { - 'name': 'lel', - 'some attribute': True - }, - { - 'name': 'lel2', - 'some attribute': True - }, - ] + def parse_resource(self, vpc): + return vpc['VpcId'], vpc + async def get_resources_in_scope(self, region): + return self.facade.get_vpcs(region) + + +class EC2Instances(ScopedResources): + def __init__(self, region): + self.region = region + self.facade = AWSFacade() + super(EC2Instances, self).__init__('instances') + + def parse_resource(self, raw_instance): + instance = {} + #instance['reservation_id'] = reservation['ReservationId'] + id = raw_instance['InstanceId'] + instance['id'] = id + instance['monitoring_enabled'] = raw_instance['Monitoring']['State'] == 'enabled' + instance['user_data'] = self.facade.get_ec2_instance_user_data( + self.region, id) + + # TODO: Those methods are slightly sketchy in my opinion (get methods which set stuff in a dictionary, say what) + get_name(raw_instance, instance, 'InstanceId') + get_keys(raw_instance, instance, ['KeyName', 'LaunchTime', 'InstanceType', 'State', 'IamInstanceProfile', 'SubnetId']) + + instance['network_interfaces'] = {} + for eni in raw_instance['NetworkInterfaces']: + nic = {} + get_keys(eni, nic, ['Association', 'Groups', 'PrivateIpAddresses', 'SubnetId', 'Ipv6Addresses']) + instance['network_interfaces'][eni['NetworkInterfaceId']] = nic + + return id, instance + + async def get_resources_in_scope(self, vpcs): + return self.facade.get_ec2_instances(self.region, vpcs) + +class EC2Images(ScopedResources): + def __init__(self, region): + self.region = region + self.facade = AWSFacade() + super(EC2Images, self).__init__('images') + + async def get_resources_in_scope(self, vpcs): + return self.facade.get_ec2_instances(self.region, vpcs) # # -*- coding: utf-8 -*- # """ From e2052add03f7368615b78e6746d170f52184f1bb Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Fri, 22 Feb 2019 10:53:03 -0500 Subject: [PATCH 048/667] Added images --- ScoutSuite/providers/aws/facade.py | 10 +++++-- ScoutSuite/providers/aws/services/ec2.py | 35 ++++++++---------------- 2 files changed, 20 insertions(+), 25 deletions(-) diff --git a/ScoutSuite/providers/aws/facade.py b/ScoutSuite/providers/aws/facade.py index 75474a336..70a540ccc 100644 --- a/ScoutSuite/providers/aws/facade.py +++ b/ScoutSuite/providers/aws/facade.py @@ -35,8 +35,7 @@ def get_ec2_instances(self, region, vpc): ec2_client = boto3.client('ec2', region_name=region) return self._get_all_pages( - lambda: ec2_client.describe_instances( - Filters=[{'Name': 'vpc-id', 'Values': [vpc]}]), + lambda: ec2_client.describe_instances(Filters=[{'Name': 'vpc-id', 'Values': [vpc]}]), lambda response: itertools.chain.from_iterable( [reservation['Instances'] for reservation in response['Reservations']]) ) @@ -48,6 +47,13 @@ def get_vpcs(self, region): lambda response: response['Vpcs'] ) + def get_ec2_images(self, region, owner_id): + vpc_client = boto3.client('ec2', region_name=region) + return self._get_all_pages( + lambda: vpc_client.describe_images(Filters=[{'Name': 'owner-id', 'Values': [owner_id]}]), + lambda response: response['Images'] + ) + def _get_all_pages(self, api_call, parse_response): resources = [] diff --git a/ScoutSuite/providers/aws/services/ec2.py b/ScoutSuite/providers/aws/services/ec2.py index 387b63697..42f8f6e53 100644 --- a/ScoutSuite/providers/aws/services/ec2.py +++ b/ScoutSuite/providers/aws/services/ec2.py @@ -2,10 +2,9 @@ from ScoutSuite.providers.aws.configs.regions_config import RegionsConfig, ScopedResources from ScoutSuite.providers.aws.facade import AWSFacade -from opinel.utils.aws import get_name +from opinel.utils.aws import get_name, get_aws_account_id from ScoutSuite.utils import get_keys, ec2_classic - class EC2(RegionsConfig): def __init__(self): super(EC2, self).__init__('ec2') @@ -16,6 +15,7 @@ async def fetch_all(self, credentials=None, regions=None, partition_name='aws'): for region in self['regions']: self['regions'][region] = await Vpcs().fetch_all(region) + self['regions'][region].update(await EC2Images(get_aws_account_id(credentials)).fetch_all(region)) class Vpcs(ScopedResources): @@ -50,7 +50,6 @@ def __init__(self, region): def parse_resource(self, raw_instance): instance = {} - #instance['reservation_id'] = reservation['ReservationId'] id = raw_instance['InstanceId'] instance['id'] = id instance['monitoring_enabled'] = raw_instance['Monitoring']['State'] == 'enabled' @@ -73,13 +72,19 @@ async def get_resources_in_scope(self, vpcs): return self.facade.get_ec2_instances(self.region, vpcs) class EC2Images(ScopedResources): - def __init__(self, region): - self.region = region + def __init__(self, owner_id): + self.owner_id = owner_id self.facade = AWSFacade() super(EC2Images, self).__init__('images') - async def get_resources_in_scope(self, vpcs): - return self.facade.get_ec2_instances(self.region, vpcs) + async def get_resources_in_scope(self, region): + return self.facade.get_ec2_images(region, self.owner_id) + + def parse_resource(self, raw_image): + raw_image['id'] = raw_image['ImageId'] + raw_image['name'] = raw_image['Name'] + + return raw_image['id'], raw_image # # -*- coding: utf-8 -*- # """ @@ -163,22 +168,6 @@ async def get_resources_in_scope(self, vpcs): # return base64.b64decode(user_data_response['UserData']['Value']).decode('utf-8') -# def parse_image(self, global_params, region, image): -# """ -# Parses a single AMI (Amazon Machine Image) - -# :param global_params: Parameters shared for all regions -# :param region: Name of the AWS region -# :param snapshot: Single image -# """ -# id = image['ImageId'] -# name = image['Name'] - -# image['id'] = id -# image['name'] = name - -# self.images[id] = image - # def parse_security_group(self, global_params, region, group): # """ # Parse a single Redsfhit security group From cea1feb51ac2146e52b5b954a38ba21b7d7a0a04 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Fri, 22 Feb 2019 15:08:26 -0500 Subject: [PATCH 049/667] Flattened scoped services one level, moved different resources in separate files --- ...ices.ec2.regions.id.vpcs.id.instances.html | 1 - .../providers/aws/configs/regions_config.py | 11 +- ScoutSuite/providers/aws/configs/services.py | 2 +- .../providers/aws/services/awslambda.py | 4 +- ScoutSuite/providers/aws/services/ec2/ami.py | 16 +++ .../providers/aws/services/ec2/instances.py | 33 +++++ .../aws/services/{ec2.py => ec2/service.py} | 129 ++---------------- ScoutSuite/providers/aws/services/ec2/vpcs.py | 24 ++++ 8 files changed, 92 insertions(+), 128 deletions(-) create mode 100644 ScoutSuite/providers/aws/services/ec2/ami.py create mode 100644 ScoutSuite/providers/aws/services/ec2/instances.py rename ScoutSuite/providers/aws/services/{ec2.py => ec2/service.py} (71%) create mode 100644 ScoutSuite/providers/aws/services/ec2/vpcs.py diff --git a/ScoutSuite/output/data/html/partials/aws/services.ec2.regions.id.vpcs.id.instances.html b/ScoutSuite/output/data/html/partials/aws/services.ec2.regions.id.vpcs.id.instances.html index c176cb407..adf65ea43 100644 --- a/ScoutSuite/output/data/html/partials/aws/services.ec2.regions.id.vpcs.id.instances.html +++ b/ScoutSuite/output/data/html/partials/aws/services.ec2.regions.id.vpcs.id.instances.html @@ -9,7 +9,6 @@

Information

  • Region: {{region}}
  • VPC: {{get_value_at 'services.ec2.regions' region 'vpcs' vpc 'name'}} ({{vpc}})
  • ID: {{id}}
    • -
  • Reservation ID: {{reservation_id}}
  • Monitoring: {{convert_bool_to_enabled monitoring_enabled}}
  • Access Key name: {{KeyName}}
  • State: {{make_title State.Name}}
    • diff --git a/ScoutSuite/providers/aws/configs/regions_config.py b/ScoutSuite/providers/aws/configs/regions_config.py index 079e6fe2e..d86eac8f9 100644 --- a/ScoutSuite/providers/aws/configs/regions_config.py +++ b/ScoutSuite/providers/aws/configs/regions_config.py @@ -4,7 +4,7 @@ import abc -class RegionsConfig(Resources): +class Regions(Resources): def __init__(self, service): self._service = service @@ -20,17 +20,12 @@ async def fetch_all(self, chosen_regions=None, partition_name='aws'): class ScopedResources(Resources): - def __init__(self, key): - self.key = key - async def fetch_all(self, scope): - resources = {} for raw_resource in await self.get_resources_in_scope(scope): name, ressource = self.parse_resource(raw_resource) - resources[name] = ressource + self[name] = ressource - self[self.key + '_count'] = len(resources) - self[self.key] = resources + self.count = len(self) return self @abc.abstractclassmethod diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index 8c3f7ed31..17eea09da 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -7,7 +7,7 @@ from ScoutSuite.providers.aws.services.cloudtrail import CloudTrailConfig from ScoutSuite.providers.aws.services.cloudwatch import CloudWatchConfig from ScoutSuite.providers.aws.services.directconnect import DirectConnectConfig -from ScoutSuite.providers.aws.services.ec2 import EC2 +from ScoutSuite.providers.aws.services.ec2.service import EC2 from ScoutSuite.providers.aws.services.efs import EFSConfig from ScoutSuite.providers.aws.services.elasticache import ElastiCacheConfig from ScoutSuite.providers.aws.services.elb import ELBConfig diff --git a/ScoutSuite/providers/aws/services/awslambda.py b/ScoutSuite/providers/aws/services/awslambda.py index b3c4d6fd4..0f4874d49 100644 --- a/ScoutSuite/providers/aws/services/awslambda.py +++ b/ScoutSuite/providers/aws/services/awslambda.py @@ -1,11 +1,11 @@ # -*- coding: utf-8 -*- from ScoutSuite.providers.base.configs.resources import Resources -from ScoutSuite.providers.aws.configs.regions_config import RegionsConfig +from ScoutSuite.providers.aws.configs.regions_config import Regions from ScoutSuite.providers.aws.facade import AWSFacade from opinel.utils.aws import build_region_list -class Lambdas(RegionsConfig): +class Lambdas(Regions): def __init__(self): super(Lambdas, self).__init__('lambda') diff --git a/ScoutSuite/providers/aws/services/ec2/ami.py b/ScoutSuite/providers/aws/services/ec2/ami.py new file mode 100644 index 000000000..437633119 --- /dev/null +++ b/ScoutSuite/providers/aws/services/ec2/ami.py @@ -0,0 +1,16 @@ +from ScoutSuite.providers.aws.configs.regions_config import ScopedResources +from ScoutSuite.providers.aws.facade import AWSFacade + +class AmazonMachineImages(ScopedResources): + def __init__(self, owner_id): + self.owner_id = owner_id + self.facade = AWSFacade() + + async def get_resources_in_scope(self, region): + return self.facade.get_ec2_images(region, self.owner_id) + + def parse_resource(self, raw_image): + raw_image['id'] = raw_image['ImageId'] + raw_image['name'] = raw_image['Name'] + + return raw_image['id'], raw_image diff --git a/ScoutSuite/providers/aws/services/ec2/instances.py b/ScoutSuite/providers/aws/services/ec2/instances.py new file mode 100644 index 000000000..1940e60dc --- /dev/null +++ b/ScoutSuite/providers/aws/services/ec2/instances.py @@ -0,0 +1,33 @@ +from ScoutSuite.providers.aws.configs.regions_config import ScopedResources +from ScoutSuite.providers.aws.facade import AWSFacade +from opinel.utils.aws import get_name +from ScoutSuite.utils import get_keys + + +class EC2Instances(ScopedResources): + def __init__(self, region): + self.region = region + self.facade = AWSFacade() + + def parse_resource(self, raw_instance): + instance = {} + id = raw_instance['InstanceId'] + instance['id'] = id + instance['monitoring_enabled'] = raw_instance['Monitoring']['State'] == 'enabled' + instance['user_data'] = self.facade.get_ec2_instance_user_data( + self.region, id) + + # TODO: Those methods are slightly sketchy in my opinion (get methods which set stuff in a dictionary, say what) + get_name(raw_instance, instance, 'InstanceId') + get_keys(raw_instance, instance, ['KeyName', 'LaunchTime', 'InstanceType', 'State', 'IamInstanceProfile', 'SubnetId']) + + instance['network_interfaces'] = {} + for eni in raw_instance['NetworkInterfaces']: + nic = {} + get_keys(eni, nic, ['Association', 'Groups', 'PrivateIpAddresses', 'SubnetId', 'Ipv6Addresses']) + instance['network_interfaces'][eni['NetworkInterfaceId']] = nic + + return id, instance + + async def get_resources_in_scope(self, vpcs): + return self.facade.get_ec2_instances(self.region, vpcs) \ No newline at end of file diff --git a/ScoutSuite/providers/aws/services/ec2.py b/ScoutSuite/providers/aws/services/ec2/service.py similarity index 71% rename from ScoutSuite/providers/aws/services/ec2.py rename to ScoutSuite/providers/aws/services/ec2/service.py index 42f8f6e53..ea34abacb 100644 --- a/ScoutSuite/providers/aws/services/ec2.py +++ b/ScoutSuite/providers/aws/services/ec2/service.py @@ -1,90 +1,31 @@ from ScoutSuite.providers.base.configs.resources import Resources -from ScoutSuite.providers.aws.configs.regions_config import RegionsConfig, ScopedResources +from ScoutSuite.providers.aws.configs.regions_config import Regions, ScopedResources from ScoutSuite.providers.aws.facade import AWSFacade +from ScoutSuite.providers.aws.services.ec2.ami import AmazonMachineImages +from ScoutSuite.providers.aws.services.ec2.vpcs import Vpcs -from opinel.utils.aws import get_name, get_aws_account_id +from opinel.utils.aws import get_aws_account_id from ScoutSuite.utils import get_keys, ec2_classic -class EC2(RegionsConfig): + +# TODO: Add docstrings + +class EC2(Regions): def __init__(self): super(EC2, self).__init__('ec2') - # TODO: Remove the credentials parameter. We had to keep it for compatibility async def fetch_all(self, credentials=None, regions=None, partition_name='aws'): await super(EC2, self).fetch_all(chosen_regions=regions, partition_name=partition_name) for region in self['regions']: - self['regions'][region] = await Vpcs().fetch_all(region) - self['regions'][region].update(await EC2Images(get_aws_account_id(credentials)).fetch_all(region)) - - -class Vpcs(ScopedResources): - def __init__(self): - self.facade = AWSFacade() - super(Vpcs, self).__init__('vpcs') - - async def fetch_all(self, region): - await super(Vpcs, self).fetch_all(region) - - for vpc in self['vpcs']: - # TODO: Add vpc_resource_types - instances = await EC2Instances(region).fetch_all(vpc) - self['vpcs'][vpc].update(instances) - - self['instances_count'] = sum([vpc['instances_count'] for vpc in self['vpcs'].values()]) - - return self - - def parse_resource(self, vpc): - return vpc['VpcId'], vpc - - async def get_resources_in_scope(self, region): - return self.facade.get_vpcs(region) - - -class EC2Instances(ScopedResources): - def __init__(self, region): - self.region = region - self.facade = AWSFacade() - super(EC2Instances, self).__init__('instances') - - def parse_resource(self, raw_instance): - instance = {} - id = raw_instance['InstanceId'] - instance['id'] = id - instance['monitoring_enabled'] = raw_instance['Monitoring']['State'] == 'enabled' - instance['user_data'] = self.facade.get_ec2_instance_user_data( - self.region, id) - - # TODO: Those methods are slightly sketchy in my opinion (get methods which set stuff in a dictionary, say what) - get_name(raw_instance, instance, 'InstanceId') - get_keys(raw_instance, instance, ['KeyName', 'LaunchTime', 'InstanceType', 'State', 'IamInstanceProfile', 'SubnetId']) + vpcs= await Vpcs().fetch_all(region) + self['regions'][region]['vpcs'] = vpcs + self['regions'][region]['instances_count'] = sum([vpc['instances'].count for vpc in self['regions'][region]['vpcs'].values()]) + self['regions'][region]['images'] = await AmazonMachineImages(get_aws_account_id(credentials)).fetch_all(region) + self['regions'][region]['images_count'] = self['regions'][region]['images'].count - instance['network_interfaces'] = {} - for eni in raw_instance['NetworkInterfaces']: - nic = {} - get_keys(eni, nic, ['Association', 'Groups', 'PrivateIpAddresses', 'SubnetId', 'Ipv6Addresses']) - instance['network_interfaces'][eni['NetworkInterfaceId']] = nic - return id, instance - async def get_resources_in_scope(self, vpcs): - return self.facade.get_ec2_instances(self.region, vpcs) - -class EC2Images(ScopedResources): - def __init__(self, owner_id): - self.owner_id = owner_id - self.facade = AWSFacade() - super(EC2Images, self).__init__('images') - - async def get_resources_in_scope(self, region): - return self.facade.get_ec2_images(region, self.owner_id) - - def parse_resource(self, raw_image): - raw_image['id'] = raw_image['ImageId'] - raw_image['name'] = raw_image['Name'] - - return raw_image['id'], raw_image # # -*- coding: utf-8 -*- # """ @@ -124,50 +65,6 @@ def parse_resource(self, raw_image): # EC2 configuration for a single AWS region # """ -# def parse_elastic_ip(self, global_params, region, eip): -# """ - -# :param global_params: -# :param region: -# :param eip: -# :return: -# """ -# self.elastic_ips[eip['PublicIp']] = eip - -# def parse_instance(self, global_params, region, reservation): -# """ -# Parse a single EC2 instance - -# :param global_params: Parameters shared for all regions -# :param region: Name of the AWS region -# :param instance: Cluster -# """ -# for i in reservation['Instances']: -# instance = {} -# vpc_id = i['VpcId'] if 'VpcId' in i and i['VpcId'] else ec2_classic -# manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types)) -# instance['reservation_id'] = reservation['ReservationId'] -# instance['id'] = i['InstanceId'] -# instance['monitoring_enabled'] = i['Monitoring']['State'] == 'enabled' -# instance['user_data'] = self._get_user_data(region, instance['id']) -# get_name(i, instance, 'InstanceId') -# get_keys(i, instance, ['KeyName', 'LaunchTime', 'InstanceType', 'State', 'IamInstanceProfile', 'SubnetId']) -# # Network interfaces & security groups -# manage_dictionary(instance, 'network_interfaces', {}) -# for eni in i['NetworkInterfaces']: -# nic = {} -# get_keys(eni, nic, ['Association', 'Groups', 'PrivateIpAddresses', 'SubnetId', 'Ipv6Addresses']) -# instance['network_interfaces'][eni['NetworkInterfaceId']] = nic -# self.vpcs[vpc_id].instances[i['InstanceId']] = instance - -# def _get_user_data(self, region, instance_id): -# user_data_response = api_clients[region].describe_instance_attribute(Attribute='userData', InstanceId=instance_id) - -# if 'Value' not in user_data_response['UserData'].keys(): -# return None - -# return base64.b64decode(user_data_response['UserData']['Value']).decode('utf-8') - # def parse_security_group(self, global_params, region, group): # """ # Parse a single Redsfhit security group diff --git a/ScoutSuite/providers/aws/services/ec2/vpcs.py b/ScoutSuite/providers/aws/services/ec2/vpcs.py new file mode 100644 index 000000000..0a57cbb80 --- /dev/null +++ b/ScoutSuite/providers/aws/services/ec2/vpcs.py @@ -0,0 +1,24 @@ +from ScoutSuite.providers.aws.configs.regions_config import ScopedResources +from ScoutSuite.providers.aws.facade import AWSFacade +from ScoutSuite.providers.aws.services.ec2.instances import EC2Instances + + +class Vpcs(ScopedResources): + def __init__(self): + self.facade = AWSFacade() + + async def fetch_all(self, region): + await super(Vpcs, self).fetch_all(region) + + for vpc in self: + # TODO: Add vpc_resource_types + instances = await EC2Instances(region).fetch_all(vpc) + self[vpc]['instances'] = instances + + return self + + def parse_resource(self, vpc): + return vpc['VpcId'], vpc + + async def get_resources_in_scope(self, region): + return self.facade.get_vpcs(region) From 3db034fe16c47938942483c46e623de93af143b2 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Fri, 22 Feb 2019 15:16:28 -0500 Subject: [PATCH 050/667] Cleaned code a bit --- ScoutSuite/providers/aws/services/ec2/service.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ScoutSuite/providers/aws/services/ec2/service.py b/ScoutSuite/providers/aws/services/ec2/service.py index ea34abacb..274698c7c 100644 --- a/ScoutSuite/providers/aws/services/ec2/service.py +++ b/ScoutSuite/providers/aws/services/ec2/service.py @@ -18,8 +18,7 @@ async def fetch_all(self, credentials=None, regions=None, partition_name='aws'): await super(EC2, self).fetch_all(chosen_regions=regions, partition_name=partition_name) for region in self['regions']: - vpcs= await Vpcs().fetch_all(region) - self['regions'][region]['vpcs'] = vpcs + self['regions'][region]['vpcs'] = await Vpcs().fetch_all(region) self['regions'][region]['instances_count'] = sum([vpc['instances'].count for vpc in self['regions'][region]['vpcs'].values()]) self['regions'][region]['images'] = await AmazonMachineImages(get_aws_account_id(credentials)).fetch_all(region) self['regions'][region]['images_count'] = self['regions'][region]['images'].count From d0b283c9338ea2acd6c189a42ffdbd70ea0eb6aa Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Fri, 22 Feb 2019 15:16:35 -0500 Subject: [PATCH 051/667] Moved lambda service to its own folder --- ScoutSuite/providers/aws/configs/services.py | 2 +- .../aws/services/{awslambda.py => awslambda/service.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename ScoutSuite/providers/aws/services/{awslambda.py => awslambda/service.py} (100%) diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index 17eea09da..388b9dab5 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -2,7 +2,7 @@ from opinel.utils.console import printError, printException, printInfo, printDebug -from ScoutSuite.providers.aws.services.awslambda import Lambdas +from ScoutSuite.providers.aws.services.awslambda.service import Lambdas from ScoutSuite.providers.aws.services.cloudformation import CloudFormationConfig from ScoutSuite.providers.aws.services.cloudtrail import CloudTrailConfig from ScoutSuite.providers.aws.services.cloudwatch import CloudWatchConfig diff --git a/ScoutSuite/providers/aws/services/awslambda.py b/ScoutSuite/providers/aws/services/awslambda/service.py similarity index 100% rename from ScoutSuite/providers/aws/services/awslambda.py rename to ScoutSuite/providers/aws/services/awslambda/service.py From b1534ea246d550708eb238494f65d0d522949b35 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Fri, 22 Feb 2019 17:53:59 -0500 Subject: [PATCH 052/667] Split facade by service --- .../providers/aws/configs/regions_config.py | 2 +- ScoutSuite/providers/aws/facade.py | 77 ------------------- ScoutSuite/providers/aws/facade/awslambda.py | 14 ++++ ScoutSuite/providers/aws/facade/ec2.py | 40 ++++++++++ ScoutSuite/providers/aws/facade/facade.py | 28 +++++++ ScoutSuite/providers/aws/facade/utils.py | 14 ++++ .../aws/services/awslambda/service.py | 4 +- ScoutSuite/providers/aws/services/ec2/ami.py | 4 +- .../providers/aws/services/ec2/instances.py | 6 +- .../providers/aws/services/ec2/service.py | 20 ----- .../providers/aws/services/ec2/snapshots.py | 27 +++++++ ScoutSuite/providers/aws/services/ec2/vpcs.py | 4 +- 12 files changed, 133 insertions(+), 107 deletions(-) delete mode 100644 ScoutSuite/providers/aws/facade.py create mode 100644 ScoutSuite/providers/aws/facade/awslambda.py create mode 100644 ScoutSuite/providers/aws/facade/ec2.py create mode 100644 ScoutSuite/providers/aws/facade/facade.py create mode 100644 ScoutSuite/providers/aws/facade/utils.py create mode 100644 ScoutSuite/providers/aws/services/ec2/snapshots.py diff --git a/ScoutSuite/providers/aws/configs/regions_config.py b/ScoutSuite/providers/aws/configs/regions_config.py index d86eac8f9..6169d5a99 100644 --- a/ScoutSuite/providers/aws/configs/regions_config.py +++ b/ScoutSuite/providers/aws/configs/regions_config.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- from ScoutSuite.providers.base.configs.resources import Resources -from ScoutSuite.providers.aws.facade import AWSFacade +from ScoutSuite.providers.aws.facade.facade import AWSFacade import abc diff --git a/ScoutSuite/providers/aws/facade.py b/ScoutSuite/providers/aws/facade.py deleted file mode 100644 index 70a540ccc..000000000 --- a/ScoutSuite/providers/aws/facade.py +++ /dev/null @@ -1,77 +0,0 @@ -import boto3 -from botocore.session import Session -from collections import Counter -import itertools -import base64 - -# TODO: It would be interesting to split the facade in different sub-facades. For example, a call could look like this: -# facade.ec2.get_instances(region) or -# facade.lambda.get_functions(region) - - -class AWSFacade(object): - async def build_region_list(self, service, chosen_regions=None, partition_name='aws'): - service = 'ec2containerservice' if service == 'ecs' else service - available_services = Session().get_available_services() - - if service not in available_services: - raise Exception('Service ' + service + ' is not available.') - - regions = Session().get_available_regions(service, partition_name) - - if chosen_regions: - return list((Counter(regions) & Counter(chosen_regions)).elements()) - else: - return regions - - def get_lambda_functions(self, region): - aws_lambda = boto3.client('lambda', region_name=region) - return self._get_all_pages( - lambda: aws_lambda.list_functions(), - lambda response: response['Functions'] - ) - - def get_ec2_instances(self, region, vpc): - ec2_client = boto3.client('ec2', region_name=region) - - return self._get_all_pages( - lambda: ec2_client.describe_instances(Filters=[{'Name': 'vpc-id', 'Values': [vpc]}]), - lambda response: itertools.chain.from_iterable( - [reservation['Instances'] for reservation in response['Reservations']]) - ) - - def get_vpcs(self, region): - vpc_client = boto3.client('ec2', region_name=region) - return self._get_all_pages( - lambda: vpc_client.describe_vpcs(), - lambda response: response['Vpcs'] - ) - - def get_ec2_images(self, region, owner_id): - vpc_client = boto3.client('ec2', region_name=region) - return self._get_all_pages( - lambda: vpc_client.describe_images(Filters=[{'Name': 'owner-id', 'Values': [owner_id]}]), - lambda response: response['Images'] - ) - - def _get_all_pages(self, api_call, parse_response): - resources = [] - - while True: - response = api_call() - - resources.extend(parse_response(response)) - marker = response.get('NextMarker', None) - if marker is None: - break - return resources - - def get_ec2_instance_user_data(self, region, instance_id): - ec2_client = boto3.client('ec2', region_name=region) - user_data_response = ec2_client.describe_instance_attribute( - Attribute='userData', InstanceId=instance_id) - - if 'Value' not in user_data_response['UserData'].keys(): - return None - - return base64.b64decode(user_data_response['UserData']['Value']).decode('utf-8') diff --git a/ScoutSuite/providers/aws/facade/awslambda.py b/ScoutSuite/providers/aws/facade/awslambda.py new file mode 100644 index 000000000..711dc0fb8 --- /dev/null +++ b/ScoutSuite/providers/aws/facade/awslambda.py @@ -0,0 +1,14 @@ +import boto3 +import base64 +import itertools + +from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils + + +class LambdaFacade: + def get_functions(self, region): + aws_lambda = boto3.client('lambda', region_name=region) + return AWSFacadeUtils.get_all_pages( + lambda: aws_lambda.list_functions(), + lambda response: response['Functions'] + ) diff --git a/ScoutSuite/providers/aws/facade/ec2.py b/ScoutSuite/providers/aws/facade/ec2.py new file mode 100644 index 000000000..877b18e2e --- /dev/null +++ b/ScoutSuite/providers/aws/facade/ec2.py @@ -0,0 +1,40 @@ +import boto3 +import base64 +import itertools + +from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils + + +class EC2Facade: + def get_instance_user_data(self, region, instance_id): + ec2_client = boto3.client('ec2', region_name=region) + user_data_response = ec2_client.describe_instance_attribute( + Attribute='userData', InstanceId=instance_id) + + if 'Value' not in user_data_response['UserData'].keys(): + return None + + return base64.b64decode(user_data_response['UserData']['Value']).decode('utf-8') + + def get_instances(self, region, vpc): + ec2_client = boto3.client('ec2', region_name=region) + + return AWSFacadeUtils.get_all_pages( + lambda: ec2_client.describe_instances(Filters=[{'Name': 'vpc-id', 'Values': [vpc]}]), + lambda response: itertools.chain.from_iterable( + [reservation['Instances'] for reservation in response['Reservations']]) + ) + + def get_vpcs(self, region): + vpc_client = boto3.client('ec2', region_name=region) + return AWSFacadeUtils.get_all_pages( + lambda: vpc_client.describe_vpcs(), + lambda response: response['Vpcs'] + ) + + def get_images(self, region, owner_id): + vpc_client = boto3.client('ec2', region_name=region) + return AWSFacadeUtils.get_all_pages( + lambda: vpc_client.describe_images(Filters=[{'Name': 'owner-id', 'Values': [owner_id]}]), + lambda response: response['Images'] + ) \ No newline at end of file diff --git a/ScoutSuite/providers/aws/facade/facade.py b/ScoutSuite/providers/aws/facade/facade.py new file mode 100644 index 000000000..7f7f3b3b4 --- /dev/null +++ b/ScoutSuite/providers/aws/facade/facade.py @@ -0,0 +1,28 @@ +from botocore.session import Session +from collections import Counter +from ScoutSuite.providers.aws.facade.ec2 import EC2Facade +from ScoutSuite.providers.aws.facade.awslambda import LambdaFacade + +class AWSFacade(object): + def __init__(self): + self.ec2 = EC2Facade() + self.awslambda = LambdaFacade() + + + async def build_region_list(self, service, chosen_regions=None, partition_name='aws'): + service = 'ec2containerservice' if service == 'ecs' else service + available_services = Session().get_available_services() + + if service not in available_services: + raise Exception('Service ' + service + ' is not available.') + + regions = Session().get_available_regions(service, partition_name) + + if chosen_regions: + return list((Counter(regions) & Counter(chosen_regions)).elements()) + else: + return regions + + + + \ No newline at end of file diff --git a/ScoutSuite/providers/aws/facade/utils.py b/ScoutSuite/providers/aws/facade/utils.py new file mode 100644 index 000000000..df2656a53 --- /dev/null +++ b/ScoutSuite/providers/aws/facade/utils.py @@ -0,0 +1,14 @@ +class AWSFacadeUtils: + @staticmethod + def get_all_pages(api_call, parse_response): + resources = [] + + while True: + response = api_call() + + resources.extend(parse_response(response)) + marker = response.get('NextMarker', None) + if marker is None: + break + + return resources \ No newline at end of file diff --git a/ScoutSuite/providers/aws/services/awslambda/service.py b/ScoutSuite/providers/aws/services/awslambda/service.py index 0f4874d49..28363222a 100644 --- a/ScoutSuite/providers/aws/services/awslambda/service.py +++ b/ScoutSuite/providers/aws/services/awslambda/service.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from ScoutSuite.providers.base.configs.resources import Resources from ScoutSuite.providers.aws.configs.regions_config import Regions -from ScoutSuite.providers.aws.facade import AWSFacade +from ScoutSuite.providers.aws.facade.facade import AWSFacade from opinel.utils.aws import build_region_list @@ -26,7 +26,7 @@ async def fetch_all(self, region): facade = AWSFacade() functions = {} - for raw_function in facade.get_lambda_functions(region): + for raw_function in facade.awslambda.get_functions(region): name, function = self.parse_function(raw_function) functions[name] = function diff --git a/ScoutSuite/providers/aws/services/ec2/ami.py b/ScoutSuite/providers/aws/services/ec2/ami.py index 437633119..ba04f58e2 100644 --- a/ScoutSuite/providers/aws/services/ec2/ami.py +++ b/ScoutSuite/providers/aws/services/ec2/ami.py @@ -1,5 +1,5 @@ from ScoutSuite.providers.aws.configs.regions_config import ScopedResources -from ScoutSuite.providers.aws.facade import AWSFacade +from ScoutSuite.providers.aws.facade.facade import AWSFacade class AmazonMachineImages(ScopedResources): def __init__(self, owner_id): @@ -7,7 +7,7 @@ def __init__(self, owner_id): self.facade = AWSFacade() async def get_resources_in_scope(self, region): - return self.facade.get_ec2_images(region, self.owner_id) + return self.facade.ec2.get_images(region, self.owner_id) def parse_resource(self, raw_image): raw_image['id'] = raw_image['ImageId'] diff --git a/ScoutSuite/providers/aws/services/ec2/instances.py b/ScoutSuite/providers/aws/services/ec2/instances.py index 1940e60dc..9fbd711f7 100644 --- a/ScoutSuite/providers/aws/services/ec2/instances.py +++ b/ScoutSuite/providers/aws/services/ec2/instances.py @@ -1,5 +1,5 @@ from ScoutSuite.providers.aws.configs.regions_config import ScopedResources -from ScoutSuite.providers.aws.facade import AWSFacade +from ScoutSuite.providers.aws.facade.facade import AWSFacade from opinel.utils.aws import get_name from ScoutSuite.utils import get_keys @@ -14,7 +14,7 @@ def parse_resource(self, raw_instance): id = raw_instance['InstanceId'] instance['id'] = id instance['monitoring_enabled'] = raw_instance['Monitoring']['State'] == 'enabled' - instance['user_data'] = self.facade.get_ec2_instance_user_data( + instance['user_data'] = self.facade.ec2.get_instance_user_data( self.region, id) # TODO: Those methods are slightly sketchy in my opinion (get methods which set stuff in a dictionary, say what) @@ -30,4 +30,4 @@ def parse_resource(self, raw_instance): return id, instance async def get_resources_in_scope(self, vpcs): - return self.facade.get_ec2_instances(self.region, vpcs) \ No newline at end of file + return self.facade.ec2.get_instances(self.region, vpcs) \ No newline at end of file diff --git a/ScoutSuite/providers/aws/services/ec2/service.py b/ScoutSuite/providers/aws/services/ec2/service.py index 274698c7c..50cd65551 100644 --- a/ScoutSuite/providers/aws/services/ec2/service.py +++ b/ScoutSuite/providers/aws/services/ec2/service.py @@ -1,6 +1,5 @@ from ScoutSuite.providers.base.configs.resources import Resources from ScoutSuite.providers.aws.configs.regions_config import Regions, ScopedResources -from ScoutSuite.providers.aws.facade import AWSFacade from ScoutSuite.providers.aws.services.ec2.ami import AmazonMachineImages from ScoutSuite.providers.aws.services.ec2.vpcs import Vpcs @@ -24,8 +23,6 @@ async def fetch_all(self, credentials=None, regions=None, partition_name='aws'): self['regions'][region]['images_count'] = self['regions'][region]['images'].count - - # # -*- coding: utf-8 -*- # """ # EC2-related classes and functions @@ -129,23 +126,6 @@ async def fetch_all(self, credentials=None, regions=None, partition_name='aws'): # return protocols, rules_count -# def parse_snapshot(self, global_params, region, snapshot): -# """ - -# :param global_params: Parameters shared for all regions -# :param region: Name of the AWS region -# :param snapshot: Single snapshot -# :return: -# """ -# snapshot['id'] = snapshot.pop('SnapshotId') -# snapshot['name'] = get_name(snapshot, snapshot, 'id') -# self.snapshots[snapshot['id']] = snapshot -# # Get snapshot attribute -# snapshot['createVolumePermission'] = \ -# api_clients[region].describe_snapshot_attribute(Attribute='createVolumePermission', SnapshotId=snapshot['id'])[ -# 'CreateVolumePermissions'] -# snapshot['public'] = self._is_public(snapshot) - # def _is_public(self, snapshot): # return any([permission.get('Group') == 'all' for permission in snapshot['createVolumePermission']]) diff --git a/ScoutSuite/providers/aws/services/ec2/snapshots.py b/ScoutSuite/providers/aws/services/ec2/snapshots.py new file mode 100644 index 000000000..d08283e40 --- /dev/null +++ b/ScoutSuite/providers/aws/services/ec2/snapshots.py @@ -0,0 +1,27 @@ +from ScoutSuite.providers.aws.configs.regions_config import ScopedResources +from ScoutSuite.providers.aws.facade.facade import AWSFacade + +class Snapshots(ScopedResources): + def __init__(self, owner_id): + self.owner_id = owner_id + self.facade = AWSFacade() + + async def get_resources_in_scope(self, region): + return self.facade.ec2.get_images(region, self.owner_id) + + def parse_resource(self, raw_image): + raw_image['id'] = raw_image['ImageId'] + raw_image['name'] = raw_image['Name'] + + return raw_image['id'], raw_image + + + def parse_snapshot(self, snapshot): + snapshot['id'] = snapshot.pop('SnapshotId') + # snapshot['name'] = get_name(snapshot, snapshot, 'id') + # self.snapshots[snapshot['id']] = snapshot + # # Get snapshot attribute + # snapshot['createVolumePermission'] = \ + # api_clients[region].describe_snapshot_attribute(Attribute='createVolumePermission', SnapshotId=snapshot['id'])[ + # 'CreateVolumePermissions'] + # snapshot['public'] = self._is_public(snapshot) \ No newline at end of file diff --git a/ScoutSuite/providers/aws/services/ec2/vpcs.py b/ScoutSuite/providers/aws/services/ec2/vpcs.py index 0a57cbb80..701009f43 100644 --- a/ScoutSuite/providers/aws/services/ec2/vpcs.py +++ b/ScoutSuite/providers/aws/services/ec2/vpcs.py @@ -1,5 +1,5 @@ from ScoutSuite.providers.aws.configs.regions_config import ScopedResources -from ScoutSuite.providers.aws.facade import AWSFacade +from ScoutSuite.providers.aws.facade.facade import AWSFacade from ScoutSuite.providers.aws.services.ec2.instances import EC2Instances @@ -21,4 +21,4 @@ def parse_resource(self, vpc): return vpc['VpcId'], vpc async def get_resources_in_scope(self, region): - return self.facade.get_vpcs(region) + return self.facade.ec2.get_vpcs(region) From 73e8afc0bdada1225326920fc0d9cf72a48d9baa Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Sat, 23 Feb 2019 23:06:08 -0500 Subject: [PATCH 053/667] Added security groups and volumes --- ScoutSuite/providers/aws/facade/ec2.py | 43 ++++++++-- ScoutSuite/providers/aws/facade/utils.py | 5 +- .../providers/aws/services/ec2/instances.py | 3 +- .../aws/services/ec2/securitygroups.py | 85 +++++++++++++++++++ .../providers/aws/services/ec2/service.py | 41 +++------ .../providers/aws/services/ec2/snapshots.py | 31 +++---- .../providers/aws/services/ec2/volumes.py | 18 ++++ ScoutSuite/providers/aws/services/ec2/vpcs.py | 5 +- 8 files changed, 174 insertions(+), 57 deletions(-) create mode 100644 ScoutSuite/providers/aws/services/ec2/securitygroups.py create mode 100644 ScoutSuite/providers/aws/services/ec2/volumes.py diff --git a/ScoutSuite/providers/aws/facade/ec2.py b/ScoutSuite/providers/aws/facade/ec2.py index 877b18e2e..596170e57 100644 --- a/ScoutSuite/providers/aws/facade/ec2.py +++ b/ScoutSuite/providers/aws/facade/ec2.py @@ -7,6 +7,7 @@ class EC2Facade: def get_instance_user_data(self, region, instance_id): + # TODO: We should save a list of the clients by region, as they are created. ec2_client = boto3.client('ec2', region_name=region) user_data_response = ec2_client.describe_instance_attribute( Attribute='userData', InstanceId=instance_id) @@ -25,16 +26,48 @@ def get_instances(self, region, vpc): [reservation['Instances'] for reservation in response['Reservations']]) ) + def get_security_groups(self, region, vpc): + ec2_client = boto3.client('ec2', region_name=region) + + return AWSFacadeUtils.get_all_pages( + lambda: ec2_client.describe_security_groups(Filters=[{'Name': 'vpc-id', 'Values': [vpc]}]), + lambda response: response['SecurityGroups'] + ) + def get_vpcs(self, region): - vpc_client = boto3.client('ec2', region_name=region) + ec2_client = boto3.client('ec2', region_name=region) return AWSFacadeUtils.get_all_pages( - lambda: vpc_client.describe_vpcs(), + lambda: ec2_client.describe_vpcs(), lambda response: response['Vpcs'] ) def get_images(self, region, owner_id): - vpc_client = boto3.client('ec2', region_name=region) + ec2_client = boto3.client('ec2', region_name=region) return AWSFacadeUtils.get_all_pages( - lambda: vpc_client.describe_images(Filters=[{'Name': 'owner-id', 'Values': [owner_id]}]), + lambda: ec2_client .describe_images(Filters=[{'Name': 'owner-id', 'Values': [owner_id]}]), lambda response: response['Images'] - ) \ No newline at end of file + ) + + def get_volumes(self, region): + ec2_client = boto3.client('ec2', region_name=region) + return AWSFacadeUtils.get_all_pages( + lambda: ec2_client.describe_volumes(), + lambda response: response['Volumes'] + ) + + + def get_snapshots(self, region, owner_id): + ec2_client = boto3.client('ec2', region_name=region) + snapshots = AWSFacadeUtils.get_all_pages( + lambda: ec2_client.describe_snapshots(Filters=[{'Name': 'owner-id', 'Values': [owner_id]}]), + lambda response: response['Snapshots'] + ) + + for snapshot in snapshots: + snapshot['CreateVolumePermissions'] = ec2_client.describe_snapshot_attribute( + Attribute='createVolumePermission', + SnapshotId=snapshot['SnapshotId'])['CreateVolumePermissions'] + + return snapshots + + \ No newline at end of file diff --git a/ScoutSuite/providers/aws/facade/utils.py b/ScoutSuite/providers/aws/facade/utils.py index df2656a53..3db154e2e 100644 --- a/ScoutSuite/providers/aws/facade/utils.py +++ b/ScoutSuite/providers/aws/facade/utils.py @@ -5,9 +5,10 @@ def get_all_pages(api_call, parse_response): while True: response = api_call() - resources.extend(parse_response(response)) - marker = response.get('NextMarker', None) + + # TODO: this marker should be passed to te api call. Also, some calls return a NextMarker, some return a NextToken. + marker = response.get('NextMarker', None) if marker is None: break diff --git a/ScoutSuite/providers/aws/services/ec2/instances.py b/ScoutSuite/providers/aws/services/ec2/instances.py index 9fbd711f7..52378be05 100644 --- a/ScoutSuite/providers/aws/services/ec2/instances.py +++ b/ScoutSuite/providers/aws/services/ec2/instances.py @@ -14,8 +14,7 @@ def parse_resource(self, raw_instance): id = raw_instance['InstanceId'] instance['id'] = id instance['monitoring_enabled'] = raw_instance['Monitoring']['State'] == 'enabled' - instance['user_data'] = self.facade.ec2.get_instance_user_data( - self.region, id) + instance['user_data'] = self.facade.ec2.get_instance_user_data(self.region, id) # TODO: Those methods are slightly sketchy in my opinion (get methods which set stuff in a dictionary, say what) get_name(raw_instance, instance, 'InstanceId') diff --git a/ScoutSuite/providers/aws/services/ec2/securitygroups.py b/ScoutSuite/providers/aws/services/ec2/securitygroups.py new file mode 100644 index 000000000..f005ecd90 --- /dev/null +++ b/ScoutSuite/providers/aws/services/ec2/securitygroups.py @@ -0,0 +1,85 @@ +from ScoutSuite.providers.aws.configs.regions_config import ScopedResources +from ScoutSuite.providers.aws.facade.facade import AWSFacade +from opinel.utils.aws import get_name +from ScoutSuite.utils import get_keys, ec2_classic +from opinel.utils.globals import manage_dictionary +from opinel.utils.fs import load_data + + +class SecurityGroups(ScopedResources): + icmp_message_types_dict = load_data('icmp_message_types.json', 'icmp_message_types') + + # TODO: The init could take a "scope" dictionary containing the necessary info. In this case, the owner_id and the region + def __init__(self, region): + self.region = region + self.facade = AWSFacade() + + async def get_resources_in_scope(self, vpc): + return self.facade.ec2.get_security_groups(self.region, vpc) + + def parse_resource(self, raw_security_group): + security_group = {} + security_group['name'] = raw_security_group['GroupName'] + security_group['id'] = raw_security_group['GroupId'] + security_group['description'] = raw_security_group['Description'] + security_group['owner_id'] = raw_security_group['OwnerId'] + + security_group['rules'] = {'ingress': {}, 'egress': {}} + ingress_protocols, ingress_rules_count = self.__parse_security_group_rules( + raw_security_group['IpPermissions']) + security_group['rules']['ingress']['protocols'] = ingress_protocols + security_group['rules']['ingress']['count'] = ingress_rules_count + + egress_protocols, egress_rules_count = self.__parse_security_group_rules( + raw_security_group['IpPermissionsEgress']) + security_group['rules']['egress']['protocols'] = egress_protocols + security_group['rules']['egress']['count'] = egress_rules_count + return security_group['id'], security_group + + def __parse_security_group_rules(self, rules): + protocols = {} + rules_count = 0 + for rule in rules: + ip_protocol = rule['IpProtocol'].upper() + if ip_protocol == '-1': + ip_protocol = 'ALL' + protocols = manage_dictionary(protocols, ip_protocol, {}) + protocols[ip_protocol] = manage_dictionary( + protocols[ip_protocol], 'ports', {}) + + # Save the port (single port or range) + port_value = 'N/A' + if 'FromPort' in rule and 'ToPort' in rule: + if ip_protocol == 'ICMP': + # FromPort with ICMP is the type of message + port_value = self.icmp_message_types_dict[str( + rule['FromPort'])] + elif rule['FromPort'] == rule['ToPort']: + port_value = str(rule['FromPort']) + else: + port_value = '%s-%s' % (rule['FromPort'], rule['ToPort']) + manage_dictionary(protocols[ip_protocol]['ports'], port_value, {}) + + # Save grants, values are either a CIDR or an EC2 security group + for grant in rule['UserIdGroupPairs']: + manage_dictionary( + protocols[ip_protocol]['ports'][port_value], 'security_groups', []) + protocols[ip_protocol]['ports'][port_value]['security_groups'].append( + grant) + rules_count = rules_count + 1 + for grant in rule['IpRanges']: + manage_dictionary( + protocols[ip_protocol]['ports'][port_value], 'cidrs', []) + protocols[ip_protocol]['ports'][port_value]['cidrs'].append( + {'CIDR': grant['CidrIp']}) + rules_count = rules_count + 1 + + # IPv6 + for grant in rule['Ipv6Ranges']: + manage_dictionary( + protocols[ip_protocol]['ports'][port_value], 'cidrs', []) + protocols[ip_protocol]['ports'][port_value]['cidrs'].append( + {'CIDR': grant['CidrIpv6']}) + rules_count = rules_count + 1 + + return protocols, rules_count diff --git a/ScoutSuite/providers/aws/services/ec2/service.py b/ScoutSuite/providers/aws/services/ec2/service.py index 50cd65551..d8b80f16a 100644 --- a/ScoutSuite/providers/aws/services/ec2/service.py +++ b/ScoutSuite/providers/aws/services/ec2/service.py @@ -2,6 +2,8 @@ from ScoutSuite.providers.aws.configs.regions_config import Regions, ScopedResources from ScoutSuite.providers.aws.services.ec2.ami import AmazonMachineImages from ScoutSuite.providers.aws.services.ec2.vpcs import Vpcs +from ScoutSuite.providers.aws.services.ec2.snapshots import Snapshots +from ScoutSuite.providers.aws.services.ec2.volumes import Volumes from opinel.utils.aws import get_aws_account_id from ScoutSuite.utils import get_keys, ec2_classic @@ -16,12 +18,22 @@ def __init__(self): async def fetch_all(self, credentials=None, regions=None, partition_name='aws'): await super(EC2, self).fetch_all(chosen_regions=regions, partition_name=partition_name) + # TODO: Is there a way to generalize this? for region in self['regions']: self['regions'][region]['vpcs'] = await Vpcs().fetch_all(region) self['regions'][region]['instances_count'] = sum([vpc['instances'].count for vpc in self['regions'][region]['vpcs'].values()]) + self['regions'][region]['security_groups_count'] = sum([vpc['security_groups'].count for vpc in self['regions'][region]['vpcs'].values()]) + self['regions'][region]['images'] = await AmazonMachineImages(get_aws_account_id(credentials)).fetch_all(region) self['regions'][region]['images_count'] = self['regions'][region]['images'].count + self['regions'][region]['snapshots'] = await Snapshots(get_aws_account_id(credentials)).fetch_all(region) + self['regions'][region]['snapshots_count'] = self['regions'][region]['snapshots'].count + + self['regions'][region]['volumes'] = await Volumes().fetch_all(region) + self['regions'][region]['volumes_count'] = self['regions'][region]['volumes'].count + + # # -*- coding: utf-8 -*- # """ @@ -126,22 +138,6 @@ async def fetch_all(self, credentials=None, regions=None, partition_name='aws'): # return protocols, rules_count -# def _is_public(self, snapshot): -# return any([permission.get('Group') == 'all' for permission in snapshot['createVolumePermission']]) - -# def parse_volume(self, global_params, region, volume): -# """ - -# :param global_params: Parameters shared for all regions -# :param region: Name of the AWS region -# :param volume: Single EBS volume -# :return: -# """ -# volume['id'] = volume.pop('VolumeId') -# volume['name'] = get_name(volume, volume, 'id') -# self.volumes[volume['id']] = volume - - # ######################################## # # EC2Config # ######################################## @@ -263,16 +259,3 @@ async def fetch_all(self, credentials=None, regions=None, partition_name='aws'): # manage_dictionary(region_info['vpcs'][vpc]['security_groups'][sg_id]['instances'], state, []) # region_info['vpcs'][vpc]['security_groups'][sg_id]['instances'][state].append(instance) - -# def manage_vpc(vpc_info, vpc_id): -# """ -# Ensure name and ID are set - -# :param vpc_info: -# :param vpc_id: -# :return: -# """ -# manage_dictionary(vpc_info, vpc_id, {}) -# vpc_info[vpc_id]['id'] = vpc_id -# if not 'name' in vpc_info[vpc_id]: -# vpc_info[vpc_id]['name'] = vpc_id diff --git a/ScoutSuite/providers/aws/services/ec2/snapshots.py b/ScoutSuite/providers/aws/services/ec2/snapshots.py index d08283e40..d3a32755b 100644 --- a/ScoutSuite/providers/aws/services/ec2/snapshots.py +++ b/ScoutSuite/providers/aws/services/ec2/snapshots.py @@ -1,27 +1,24 @@ from ScoutSuite.providers.aws.configs.regions_config import ScopedResources from ScoutSuite.providers.aws.facade.facade import AWSFacade +from opinel.utils.aws import get_name + class Snapshots(ScopedResources): + + # TODO: The init could take a "scope" dictionary containing the necessary info. In this case, the owner_id and the region def __init__(self, owner_id): self.owner_id = owner_id self.facade = AWSFacade() - - async def get_resources_in_scope(self, region): - return self.facade.ec2.get_images(region, self.owner_id) - - def parse_resource(self, raw_image): - raw_image['id'] = raw_image['ImageId'] - raw_image['name'] = raw_image['Name'] - return raw_image['id'], raw_image + async def get_resources_in_scope(self, region): + return self.facade.ec2.get_snapshots(region, self.owner_id) + def parse_resource(self, raw_snapshot): + raw_snapshot['id'] = raw_snapshot.pop('SnapshotId') + raw_snapshot['name'] = get_name(raw_snapshot, raw_snapshot, 'id') + raw_snapshot['public'] = self._is_public(raw_snapshot) + return raw_snapshot['id'], raw_snapshot - def parse_snapshot(self, snapshot): - snapshot['id'] = snapshot.pop('SnapshotId') - # snapshot['name'] = get_name(snapshot, snapshot, 'id') - # self.snapshots[snapshot['id']] = snapshot - # # Get snapshot attribute - # snapshot['createVolumePermission'] = \ - # api_clients[region].describe_snapshot_attribute(Attribute='createVolumePermission', SnapshotId=snapshot['id'])[ - # 'CreateVolumePermissions'] - # snapshot['public'] = self._is_public(snapshot) \ No newline at end of file + @staticmethod + def _is_public(snapshot): + return any([permission.get('Group') == 'all' for permission in snapshot['CreateVolumePermissions']]) diff --git a/ScoutSuite/providers/aws/services/ec2/volumes.py b/ScoutSuite/providers/aws/services/ec2/volumes.py new file mode 100644 index 000000000..d010ea24d --- /dev/null +++ b/ScoutSuite/providers/aws/services/ec2/volumes.py @@ -0,0 +1,18 @@ +from ScoutSuite.providers.aws.configs.regions_config import ScopedResources +from ScoutSuite.providers.aws.facade.facade import AWSFacade +from opinel.utils.aws import get_name + + +class Volumes(ScopedResources): + + # TODO: The init could take a "scope" dictionary containing the necessary info. In this case, the owner_id and the region + def __init__(self): + self.facade = AWSFacade() + + async def get_resources_in_scope(self, region): + return self.facade.ec2.get_volumes(region) + + def parse_resource(self, raw_volume): + raw_volume['id'] = raw_volume.pop('VolumeId') + raw_volume['name'] = get_name(raw_volume, raw_volume, 'id') + return raw_volume['id'], raw_volume diff --git a/ScoutSuite/providers/aws/services/ec2/vpcs.py b/ScoutSuite/providers/aws/services/ec2/vpcs.py index 701009f43..71d742150 100644 --- a/ScoutSuite/providers/aws/services/ec2/vpcs.py +++ b/ScoutSuite/providers/aws/services/ec2/vpcs.py @@ -1,6 +1,7 @@ from ScoutSuite.providers.aws.configs.regions_config import ScopedResources from ScoutSuite.providers.aws.facade.facade import AWSFacade from ScoutSuite.providers.aws.services.ec2.instances import EC2Instances +from ScoutSuite.providers.aws.services.ec2.securitygroups import SecurityGroups class Vpcs(ScopedResources): @@ -12,8 +13,8 @@ async def fetch_all(self, region): for vpc in self: # TODO: Add vpc_resource_types - instances = await EC2Instances(region).fetch_all(vpc) - self[vpc]['instances'] = instances + self[vpc]['instances'] = await EC2Instances(region).fetch_all(vpc) + self[vpc]['security_groups'] = await SecurityGroups(region).fetch_all(vpc) return self From 82b50258fa39619201db4e8684ead771780039cf Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Sat, 23 Feb 2019 23:26:50 -0500 Subject: [PATCH 054/667] Removed copied code --- .../providers/aws/services/ec2/service.py | 73 ------------------- 1 file changed, 73 deletions(-) diff --git a/ScoutSuite/providers/aws/services/ec2/service.py b/ScoutSuite/providers/aws/services/ec2/service.py index d8b80f16a..8fe478678 100644 --- a/ScoutSuite/providers/aws/services/ec2/service.py +++ b/ScoutSuite/providers/aws/services/ec2/service.py @@ -56,14 +56,6 @@ async def fetch_all(self, credentials=None, regions=None, partition_name='aws'): # from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients -# ######################################## -# # Globals -# ######################################## - -# icmp_message_types_dict = load_data('icmp_message_types.json', 'icmp_message_types') -# protocols_dict = load_data('protocols.json', 'protocols') - - # ######################################## # # EC2RegionConfig # ######################################## @@ -73,71 +65,6 @@ async def fetch_all(self, credentials=None, regions=None, partition_name='aws'): # EC2 configuration for a single AWS region # """ -# def parse_security_group(self, global_params, region, group): -# """ -# Parse a single Redsfhit security group - -# :param global_params: Parameters shared for all regions -# :param region: Name of the AWS region -# :param security)_group: Security group -# """ -# vpc_id = group['VpcId'] if 'VpcId' in group and group['VpcId'] else ec2_classic -# manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types)) -# security_group = {} -# security_group['name'] = group['GroupName'] -# security_group['id'] = group['GroupId'] -# security_group['description'] = group['Description'] -# security_group['owner_id'] = group['OwnerId'] -# security_group['rules'] = {'ingress': {}, 'egress': {}} -# security_group['rules']['ingress']['protocols'], security_group['rules']['ingress'][ -# 'count'] = self.__parse_security_group_rules(group['IpPermissions']) -# security_group['rules']['egress']['protocols'], security_group['rules']['egress'][ -# 'count'] = self.__parse_security_group_rules(group['IpPermissionsEgress']) -# self.vpcs[vpc_id].security_groups[group['GroupId']] = security_group - -# def __parse_security_group_rules(self, rules): -# """ - -# :param self: -# :param rules: -# :return: -# """ -# protocols = {} -# rules_count = 0 -# for rule in rules: -# ip_protocol = rule['IpProtocol'].upper() -# if ip_protocol == '-1': -# ip_protocol = 'ALL' -# protocols = manage_dictionary(protocols, ip_protocol, {}) -# protocols[ip_protocol] = manage_dictionary(protocols[ip_protocol], 'ports', {}) -# # Save the port (single port or range) -# port_value = 'N/A' -# if 'FromPort' in rule and 'ToPort' in rule: -# if ip_protocol == 'ICMP': -# # FromPort with ICMP is the type of message -# port_value = icmp_message_types_dict[str(rule['FromPort'])] -# elif rule['FromPort'] == rule['ToPort']: -# port_value = str(rule['FromPort']) -# else: -# port_value = '%s-%s' % (rule['FromPort'], rule['ToPort']) -# manage_dictionary(protocols[ip_protocol]['ports'], port_value, {}) -# # Save grants, values are either a CIDR or an EC2 security group -# for grant in rule['UserIdGroupPairs']: -# manage_dictionary(protocols[ip_protocol]['ports'][port_value], 'security_groups', []) -# protocols[ip_protocol]['ports'][port_value]['security_groups'].append(grant) -# rules_count = rules_count + 1 -# for grant in rule['IpRanges']: -# manage_dictionary(protocols[ip_protocol]['ports'][port_value], 'cidrs', []) -# protocols[ip_protocol]['ports'][port_value]['cidrs'].append({'CIDR': grant['CidrIp']}) -# rules_count = rules_count + 1 -# # IPv6 -# for grant in rule['Ipv6Ranges']: -# manage_dictionary(protocols[ip_protocol]['ports'][port_value], 'cidrs', []) -# protocols[ip_protocol]['ports'][port_value]['cidrs'].append({'CIDR': grant['CidrIpv6']}) -# rules_count = rules_count + 1 - -# return protocols, rules_count - # ######################################## # # EC2Config # ######################################## From 794b071c0d7b04c31672e7e5737026b498fcfdf7 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Sun, 24 Feb 2019 10:12:52 -0500 Subject: [PATCH 055/667] Added network interfaces --- .../providers/aws/configs/regions_config.py | 7 ++++++- ScoutSuite/providers/aws/facade/ec2.py | 7 +++++++ ScoutSuite/providers/aws/provider.py | 12 ++++++------ .../aws/services/ec2/networkinterfaces.py | 16 ++++++++++++++++ ScoutSuite/providers/aws/services/ec2/service.py | 2 ++ ScoutSuite/providers/aws/services/ec2/vpcs.py | 4 +++- 6 files changed, 40 insertions(+), 8 deletions(-) create mode 100644 ScoutSuite/providers/aws/services/ec2/networkinterfaces.py diff --git a/ScoutSuite/providers/aws/configs/regions_config.py b/ScoutSuite/providers/aws/configs/regions_config.py index 6169d5a99..5c2fedf61 100644 --- a/ScoutSuite/providers/aws/configs/regions_config.py +++ b/ScoutSuite/providers/aws/configs/regions_config.py @@ -14,7 +14,12 @@ async def fetch_all(self, chosen_regions=None, partition_name='aws'): self['regions'] = {} for region in await facade.build_region_list(self._service, chosen_regions, partition_name): - self['regions'][region] = {} + # TODO: Do we really need id, region AND name? + self['regions'][region] = { + 'id': region, + 'region': region, + 'name': region + } self['regions_count'] = len(self['regions']) diff --git a/ScoutSuite/providers/aws/facade/ec2.py b/ScoutSuite/providers/aws/facade/ec2.py index 596170e57..e27134cb6 100644 --- a/ScoutSuite/providers/aws/facade/ec2.py +++ b/ScoutSuite/providers/aws/facade/ec2.py @@ -48,6 +48,13 @@ def get_images(self, region, owner_id): lambda response: response['Images'] ) + def get_network_interfaces(self, region, vpc): + ec2_client = boto3.client('ec2', region_name=region) + return AWSFacadeUtils.get_all_pages( + lambda: ec2_client.describe_network_interfaces(Filters=[{'Name': 'vpc-id', 'Values': [vpc]}]), + lambda response: response['NetworkInterfaces'] + ) + def get_volumes(self, region): ec2_client = boto3.client('ec2', region_name=region) return AWSFacadeUtils.get_all_pages( diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index f990fa13e..4054370a4 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -79,12 +79,12 @@ def preprocessing(self, ip_ranges=None, ip_ranges_name_key=None): # Various data processing calls self._check_ec2_zone_distribution() self._add_security_group_name_to_ec2_grants() - # self._add_last_snapshot_date_to_ec2_volumes() - # self._process_cloudtrail_trails(self.services['cloudtrail']) - # self._add_cidr_display_name(ip_ranges, ip_ranges_name_key) - # self._merge_route53_and_route53domains() - # self._match_instances_and_roles() - # self._match_iam_policies_and_buckets() + self._add_last_snapshot_date_to_ec2_volumes() + self._process_cloudtrail_trails(self.services['cloudtrail']) + self._add_cidr_display_name(ip_ranges, ip_ranges_name_key) + self._merge_route53_and_route53domains() + self._match_instances_and_roles() + self._match_iam_policies_and_buckets() super(AWSProvider, self).preprocessing() diff --git a/ScoutSuite/providers/aws/services/ec2/networkinterfaces.py b/ScoutSuite/providers/aws/services/ec2/networkinterfaces.py new file mode 100644 index 000000000..731678260 --- /dev/null +++ b/ScoutSuite/providers/aws/services/ec2/networkinterfaces.py @@ -0,0 +1,16 @@ +from ScoutSuite.providers.aws.configs.regions_config import ScopedResources +from ScoutSuite.providers.aws.facade.facade import AWSFacade + + +class NetworkInterfaces(ScopedResources): + # TODO: The init could take a "scope" dictionary containing the necessary info. In this case, the owner_id and the region + def __init__(self, region): + self.region = region + self.facade = AWSFacade() + + async def get_resources_in_scope(self, vpc): + return self.facade.ec2.get_network_interfaces(self.region, vpc) + + def parse_resource(self, raw_network_interace): + raw_network_interace['name'] = raw_network_interace['NetworkInterfaceId'] + return raw_network_interace['NetworkInterfaceId'], raw_network_interace diff --git a/ScoutSuite/providers/aws/services/ec2/service.py b/ScoutSuite/providers/aws/services/ec2/service.py index 8fe478678..387e1bdbe 100644 --- a/ScoutSuite/providers/aws/services/ec2/service.py +++ b/ScoutSuite/providers/aws/services/ec2/service.py @@ -23,6 +23,7 @@ async def fetch_all(self, credentials=None, regions=None, partition_name='aws'): self['regions'][region]['vpcs'] = await Vpcs().fetch_all(region) self['regions'][region]['instances_count'] = sum([vpc['instances'].count for vpc in self['regions'][region]['vpcs'].values()]) self['regions'][region]['security_groups_count'] = sum([vpc['security_groups'].count for vpc in self['regions'][region]['vpcs'].values()]) + self['regions'][region]['network_interfaces_count'] = sum([vpc['network_interfaces'].count for vpc in self['regions'][region]['vpcs'].values()]) self['regions'][region]['images'] = await AmazonMachineImages(get_aws_account_id(credentials)).fetch_all(region) self['regions'][region]['images_count'] = self['regions'][region]['images'].count @@ -32,6 +33,7 @@ async def fetch_all(self, credentials=None, regions=None, partition_name='aws'): self['regions'][region]['volumes'] = await Volumes().fetch_all(region) self['regions'][region]['volumes_count'] = self['regions'][region]['volumes'].count + diff --git a/ScoutSuite/providers/aws/services/ec2/vpcs.py b/ScoutSuite/providers/aws/services/ec2/vpcs.py index 71d742150..48668d252 100644 --- a/ScoutSuite/providers/aws/services/ec2/vpcs.py +++ b/ScoutSuite/providers/aws/services/ec2/vpcs.py @@ -2,6 +2,7 @@ from ScoutSuite.providers.aws.facade.facade import AWSFacade from ScoutSuite.providers.aws.services.ec2.instances import EC2Instances from ScoutSuite.providers.aws.services.ec2.securitygroups import SecurityGroups +from ScoutSuite.providers.aws.services.ec2.networkinterfaces import NetworkInterfaces class Vpcs(ScopedResources): @@ -15,11 +16,12 @@ async def fetch_all(self, region): # TODO: Add vpc_resource_types self[vpc]['instances'] = await EC2Instances(region).fetch_all(vpc) self[vpc]['security_groups'] = await SecurityGroups(region).fetch_all(vpc) + self[vpc]['network_interfaces'] = await NetworkInterfaces(region).fetch_all(vpc) return self def parse_resource(self, vpc): - return vpc['VpcId'], vpc + return vpc['VpcId'], {} async def get_resources_in_scope(self, region): return self.facade.ec2.get_vpcs(region) From eb61fad6286568742ff0ae4294aead02dd8ae154 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Sun, 24 Feb 2019 10:15:26 -0500 Subject: [PATCH 056/667] Removed commented code --- .../providers/aws/services/ec2/service.py | 157 +----------------- 1 file changed, 1 insertion(+), 156 deletions(-) diff --git a/ScoutSuite/providers/aws/services/ec2/service.py b/ScoutSuite/providers/aws/services/ec2/service.py index 387e1bdbe..018ab6aca 100644 --- a/ScoutSuite/providers/aws/services/ec2/service.py +++ b/ScoutSuite/providers/aws/services/ec2/service.py @@ -32,159 +32,4 @@ async def fetch_all(self, credentials=None, regions=None, partition_name='aws'): self['regions'][region]['snapshots_count'] = self['regions'][region]['snapshots'].count self['regions'][region]['volumes'] = await Volumes().fetch_all(region) - self['regions'][region]['volumes_count'] = self['regions'][region]['volumes'].count - - - - -# # -*- coding: utf-8 -*- -# """ -# EC2-related classes and functions -# """ - -# # TODO: move a lot of this to VPCconfig, and use some sort of filter to only list SGs in EC2 classic -# import netaddr -# import base64 - -# from opinel.utils.aws import get_name -# from opinel.utils.console import printException, printInfo -# from opinel.utils.fs import load_data -# from opinel.utils.globals import manage_dictionary - -# from ScoutSuite.providers.aws.configs.vpc import VPCConfig -# from ScoutSuite.providers.base.configs.browser import get_attribute_at -# from ScoutSuite.providers.base.provider import BaseProvider -# from ScoutSuite.utils import get_keys, ec2_classic -# from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients - - -# ######################################## -# # EC2RegionConfig -# ######################################## - -# class EC2RegionConfig(RegionConfig): -# """ -# EC2 configuration for a single AWS region -# """ - -# ######################################## -# # EC2Config -# ######################################## - -# class EC2Config(RegionalServiceConfig): -# """ -# EC2 configuration for all AWS regions -# """ - -# region_config_class = EC2RegionConfig - -# def __init__(self, service_metadata, thread_config=4): -# super(EC2Config, self).__init__(service_metadata, thread_config) - - -# ######################################## -# ##### EC2 analysis functions -# ######################################## - -# def analyze_ec2_config(ec2_info, aws_account_id, force_write): -# try: -# printInfo('Analyzing EC2 config... ', newLine=False) -# # Custom EC2 analysis -# # check_for_elastic_ip(ec2_info) -# # FIXME - commented for now as this method doesn't seem to be defined anywhere' -# # list_network_attack_surface(ec2_info, 'attack_surface', 'PublicIpAddress') -# # TODO: make this optional, commented out for now -# # list_network_attack_surface(ec2_info, 'private_attack_surface', 'PrivateIpAddress') -# printInfo('Success') -# except Exception as e: -# printInfo('Error') -# printException(e) - -# def add_security_group_name_to_ec2_grants_callback(ec2_config, current_config, path, current_path, ec2_grant, -# callback_args): -# """ -# Callback - -# :param ec2_config: -# :param current_config: -# :param path: -# :param current_path: -# :param ec2_grant: -# :param callback_args: -# :return: -# """ -# sg_id = ec2_grant['GroupId'] -# if sg_id in current_path: -# target = current_path[:(current_path.index(sg_id) + 1)] -# ec2_grant['GroupName'] = get_attribute_at(ec2_config, target, 'name') -# elif ec2_grant['UserId'] == callback_args['AWSAccountId']: -# if 'VpcId' in ec2_grant: -# target = current_path[:(current_path.index('vpcs') + 1)] -# target.append(ec2_grant['VpcId']) -# target.append('security_groups') -# target.append(sg_id) -# else: -# target = current_path[:(current_path.index('security_groups') + 1)] -# target.append(sg_id) -# ec2_grant['GroupName'] = get_attribute_at(ec2_config, target, 'name') - - -# def check_for_elastic_ip(ec2_info): -# """ -# Check that the whitelisted EC2 IP addresses are not static IPs - -# :param ec2_info: -# :return: -# """ -# # Build a list of all elatic IP in the account -# elastic_ips = [] -# for region in ec2_info['regions']: -# if 'elastic_ips' in ec2_info['regions'][region]: -# for eip in ec2_info['regions'][region]['elastic_ips']: -# elastic_ips.append(eip) -# new_items = [] -# new_macro_items = [] -# for i, item in enumerate(ec2_info['violations']['non-elastic-ec2-public-ip-whitelisted'].items): -# ip = netaddr.IPNetwork(item) -# found = False -# for eip in elastic_ips: -# eip = netaddr.IPNetwork(eip) -# if ip in eip: -# found = True -# break -# if not found: -# new_items.append(ec2_info['violations']['non-elastic-ec2-public-ip-whitelisted'].items[i]) -# new_macro_items.append(ec2_info['violations']['non-elastic-ec2-public-ip-whitelisted'].macro_items[i]) -# ec2_info['violations']['non-elastic-ec2-public-ip-whitelisted'].items = new_items -# ec2_info['violations']['non-elastic-ec2-public-ip-whitelisted'].macro_items = new_macro_items - - -# def link_elastic_ips_callback2(ec2_config, current_config, path, current_path, instance_id, callback_args): -# if instance_id == callback_args['instance_id']: -# if not 'PublicIpAddress' in current_config: -# current_config['PublicIpAddress'] = callback_args['elastic_ip'] -# elif current_config['PublicIpAddress'] != callback_args['elastic_ip']: -# printInfo('Warning: public IP address exists (%s) for an instance associated with an elastic IP (%s)' % ( -# current_config['PublicIpAddress'], callback_args['elastic_ip'])) -# # This can happen... fix it - - -# def list_instances_in_security_groups(region_info): -# """ -# Once all the data has been fetched, iterate through instances and list them -# Could this be done when all the "used_by" values are set ??? TODO - -# :param region_info: -# :return: -# """ -# for vpc in region_info['vpcs']: -# if not 'instances' in region_info['vpcs'][vpc]: -# return -# for instance in region_info['vpcs'][vpc]['instances']: -# state = region_info['vpcs'][vpc]['instances'][instance]['State']['Name'] -# for sg in region_info['vpcs'][vpc]['instances'][instance]['security_groups']: -# sg_id = sg['GroupId'] -# manage_dictionary(region_info['vpcs'][vpc]['security_groups'][sg_id], 'instances', {}) -# manage_dictionary(region_info['vpcs'][vpc]['security_groups'][sg_id]['instances'], state, []) -# region_info['vpcs'][vpc]['security_groups'][sg_id]['instances'][state].append(instance) - + self['regions'][region]['volumes_count'] = self['regions'][region]['volumes'].count \ No newline at end of file From 85b15f016d6c43463dffeacafd8cfd301a6475fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 16:10:57 -0500 Subject: [PATCH 057/667] Add SimpleResources and CompositeResources abstract classes. --- .../providers/base/configs/resources.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/ScoutSuite/providers/base/configs/resources.py b/ScoutSuite/providers/base/configs/resources.py index d709c4c3c..c75c01766 100644 --- a/ScoutSuite/providers/base/configs/resources.py +++ b/ScoutSuite/providers/base/configs/resources.py @@ -8,3 +8,27 @@ class Resources(dict, metaclass=abc.ABCMeta): @abc.abstractmethod async def fetch_all(self, **kwargs): raise NotImplementedError() + + +class SimpleResources(Resources, metaclass=abc.ABCMeta): + + @abc.abstractmethod + def parse_resource(self, resource): + raise NotImplementedError + + @abc.abstractmethod + async def get_resources_from_api(self, **kwargs): + raise NotImplementedError + + +class CompositeResources(Resources, metaclass=abc.ABCMeta): + + # The following enforces that classes which inherits from CompositeResources define a `children` attribute: + @property + @abc.abstractmethod + def children(self): + raise NotImplementedError + + @abc.abstractmethod + async def fetch_children(self, **kwargs): + raise NotImplementedError From 2069f94a3e0e53b1dbbb671b1028fd05e4e6e5d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 16:12:34 -0500 Subject: [PATCH 058/667] Add AzureSimpleResources and AzureCompositeResources abstract classes. --- .../providers/azure/resources/resources.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 ScoutSuite/providers/azure/resources/resources.py diff --git a/ScoutSuite/providers/azure/resources/resources.py b/ScoutSuite/providers/azure/resources/resources.py new file mode 100644 index 000000000..36072503e --- /dev/null +++ b/ScoutSuite/providers/azure/resources/resources.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- + +import abc + +from ScoutSuite.providers.base.configs.resources import SimpleResources +from ScoutSuite.providers.base.configs.resources import CompositeResources + + +class AzureSimpleResources(SimpleResources, metaclass=abc.ABCMeta): + + async def fetch_all(self, credentials): + raw_resource = await self.get_resources_from_api(credentials) + name, resource = self.parse_resource(raw_resource) + self[name] = resource + + +class AzureCompositeResources(CompositeResources, metaclass=abc.ABCMeta): + + # TODO: get rid of the credentials. + async def fetch_children(self, parent, credentials, **kwargs): + for child_class in self.children: + child = child_class(**kwargs) + await child.fetch_all(credentials) + parent.update(child) From cf1ffa71482eb2b37cea07447b1608f419da6e09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 16:13:27 -0500 Subject: [PATCH 059/667] Make DatabaseBlobAuditingPolicies AzureSimpleResources. --- .../sql/database_blob_auditing_policies.py | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py b/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py index 743b0e8a2..b2d3d74a2 100644 --- a/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py +++ b/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py @@ -2,26 +2,21 @@ from azure.mgmt.sql import SqlManagementClient -from ScoutSuite.providers.base.configs.resources import Resources +from ..resources import AzureSimpleResources -class DatabaseBlobAuditingPolicies(Resources): +class DatabaseBlobAuditingPolicies(AzureSimpleResources): def __init__(self, resource_group_name, server_name, database_name): self.resource_group_name = resource_group_name self.server_name = server_name self.database_name = database_name - async def fetch_all(self, credentials): - # sdk container: + # TODO: make it really async. + async def get_resources_from_api(self, credentials): api = SqlManagementClient(credentials.credentials, credentials.subscription_id) + return api.database_blob_auditing_policies.get( + self.resource_group_name, self.server_name, self.database_name) - policies =\ - api.database_blob_auditing_policies.get(self.resource_group_name, self.server_name, self.database_name) - # TODO: await api.database_blob_auditing_policies.get(self.resource_group_name, self.server_name, self.database_name) - - self['auditing_enabled'] = self._is_auditing_enabled(policies) - - @staticmethod - def _is_auditing_enabled(policies): - return policies.state == "Enabled" + def parse_resource(self, policies): + return 'auditing_enabled', policies.state == "Enabled" From 08a3dded302a75c780c092d801652db7daeb5a5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 16:13:50 -0500 Subject: [PATCH 060/667] Make DatabaseThreatDetectionPolicies AzureSimpleResources. --- .../sql/database_threat_detection_policies.py | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py b/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py index 874b14307..2563f813a 100644 --- a/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py +++ b/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py @@ -2,26 +2,21 @@ from azure.mgmt.sql import SqlManagementClient -from ScoutSuite.providers.base.configs.resources import Resources +from ..resources import AzureSimpleResources -class DatabaseThreatDetectionPolicies(Resources): +class DatabaseThreatDetectionPolicies(AzureSimpleResources): def __init__(self, resource_group_name, server_name, database_name): self.resource_group_name = resource_group_name self.server_name = server_name self.database_name = database_name - async def fetch_all(self, credentials): - # sdk container: + # TODO: make it really async. + async def get_resources_from_api(self, credentials): api = SqlManagementClient(credentials.credentials, credentials.subscription_id) + return api.database_threat_detection_policies.get( + self.resource_group_name, self.server_name, self.database_name) - policies =\ - api.database_threat_detection_policies.get(self.resource_group_name, self.server_name, self.database_name) - # TODO: await api.transparent_data_encryptions.get(self.resource_group_name, self.server_name, self.database_name) - - self['threat_detection_enabled'] = self._is_threat_detection_enabled(policies) - - @staticmethod - def _is_threat_detection_enabled(policies): - return policies.state == "Enabled" + def parse_resource(self, policies): + return 'threat_detection_enabled', policies.state == "Enabled" From e331f3c7c15b979a0dd160560a69084ad6c129e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 16:14:11 -0500 Subject: [PATCH 061/667] Make ReplicationLinks AzureSimpleResources. --- .../azure/resources/sql/replication_links.py | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/replication_links.py b/ScoutSuite/providers/azure/resources/sql/replication_links.py index 20db831f9..dbedd953d 100644 --- a/ScoutSuite/providers/azure/resources/sql/replication_links.py +++ b/ScoutSuite/providers/azure/resources/sql/replication_links.py @@ -2,26 +2,21 @@ from azure.mgmt.sql import SqlManagementClient -from ScoutSuite.providers.base.configs.resources import Resources +from ..resources import AzureSimpleResources -class ReplicationLinks(Resources): +class ReplicationLinks(AzureSimpleResources): def __init__(self, resource_group_name, server_name, database_name): self.resource_group_name = resource_group_name self.server_name = server_name self.database_name = database_name - async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): - # sdk container: + # TODO: make it really async. + async def get_resources_from_api(self, credentials): api = SqlManagementClient(credentials.credentials, credentials.subscription_id) + return list(api.replication_links.list_by_database( + self.resource_group_name, self.server_name, self.database_name)) - links =\ - list(api.replication_links.list_by_database(self.resource_group_name, self.server_name, self.database_name)) - # TODO: await api.replication_links.list_by_databases(self.resource_group_name, self.server_name, self.database_name) - - self['replication_configured'] = self._is_replication_configured(links) - - @staticmethod - def _is_replication_configured(links): - return len(links) > 0 + def parse_resource(self, links): + return 'replication_configured', len(links) > 0 From 166ddef706edbfca3f8321f0bf65b581ca0e82a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 16:15:08 -0500 Subject: [PATCH 062/667] Remove unused variable. --- .../azure/resources/sql/server_azure_ad_administrators.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py b/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py index 189a3cf2d..a6de8eba7 100644 --- a/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py +++ b/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py @@ -12,14 +12,13 @@ def __init__(self, resource_group_name, server_name): self.resource_group_name = resource_group_name self.server_name = server_name + # TODO: make it really async. async def fetch_all(self, credentials): # sdk container: api = SqlManagementClient(credentials.credentials, credentials.subscription_id) try: - admins = \ - api.server_azure_ad_administrators.get(self.resource_group_name, self.server_name) - # TODO: await api.server_azure_ad_administrators.get(self.resource_group_name, self.server_name) + _ = api.server_azure_ad_administrators.get(self.resource_group_name, self.server_name) self['ad_admin_configured'] = True except CloudError: # no ad admin configured returns a 404 error self['ad_admin_configured'] = False From 98f4afe46aff6564c7b8ed31b2e100b3192d15dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 16:15:33 -0500 Subject: [PATCH 063/667] Make TransparentDataEncryptions AzureSimpleResources. --- .../sql/transparent_data_encryptions.py | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py b/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py index 95a971bf1..becc2748c 100644 --- a/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py +++ b/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py @@ -2,28 +2,21 @@ from azure.mgmt.sql import SqlManagementClient -from ScoutSuite.providers.base.configs.resources import Resources +from ..resources import AzureSimpleResources -class TransparentDataEncryptions(Resources): +class TransparentDataEncryptions(AzureSimpleResources): def __init__(self, resource_group_name, server_name, database_name): self.resource_group_name = resource_group_name self.server_name = server_name self.database_name = database_name - self.value = None - - async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): - # sdk container: + # TODO: make it really async. + async def get_resources_from_api(self, credentials): api = SqlManagementClient(credentials.credentials, credentials.subscription_id) + return api.transparent_data_encryptions.get( + self.resource_group_name, self.server_name, self.database_name) - encryptions =\ - api.transparent_data_encryptions.get(self.resource_group_name, self.server_name, self.database_name) - # TODO: await api.transparent_data_encryptions.get(self.resource_group_name, self.server_name, self.database_name) - - self['transparent_data_encryption_enabled'] = self._is_transparent_data_encryption_enabled(encryptions) - - @staticmethod - def _is_transparent_data_encryption_enabled(encryptions): - return encryptions.status == "Enabled" + def parse_resource(self, encryptions): + return 'transparent_data_encryption_enabled', encryptions.status == "Enabled" From d6b086c39e2c53833f891efd943592e0c3b45eee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 16:16:08 -0500 Subject: [PATCH 064/667] Make Databases AzureCompositeResources. --- .../azure/resources/sql/databases.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/databases.py b/ScoutSuite/providers/azure/resources/sql/databases.py index c58891f6e..8178ff3be 100644 --- a/ScoutSuite/providers/azure/resources/sql/databases.py +++ b/ScoutSuite/providers/azure/resources/sql/databases.py @@ -2,7 +2,7 @@ from azure.mgmt.sql import SqlManagementClient -from ScoutSuite.providers.base.configs.resources import Resources +from ..resources import AzureCompositeResources from .database_blob_auditing_policies import DatabaseBlobAuditingPolicies from .database_threat_detection_policies import DatabaseThreatDetectionPolicies @@ -10,7 +10,7 @@ from .transparent_data_encryptions import TransparentDataEncryptions -class Databases(Resources): +class Databases(AzureCompositeResources): children = [ DatabaseBlobAuditingPolicies, DatabaseThreatDetectionPolicies, @@ -22,19 +22,19 @@ def __init__(self, resource_group_name, server_name): self.resource_group_name = resource_group_name self.server_name = server_name + # TODO: make it really async. async def fetch_all(self, credentials): - # sdk container: api = SqlManagementClient(credentials.credentials, credentials.subscription_id) self['databases'] = {} - # TODO: for db in await api.databases.list_by_server(self.resource_group_name, self.server_name): for db in api.databases.list_by_server(self.resource_group_name, self.server_name): self['databases'][db.name] = { 'id': db.name, } - - # put the following code in a fetch_children() parent method (typical method for a composite node)? - for resources_class in self.simple_children: - resources = resources_class(self.resource_group_name, self.server_name, db.name) - await resources.fetch_all(credentials) - self['databases'][db.name].update(resources) + await self.fetch_children( + parent=self['databases'][db.name], + resource_group_name=self.resource_group_name, + server_name=self.server_name, + database_name=db.name, + credentials=credentials + ) From e0ac32bb1f08c559769714397ffc05d4a84f1cfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 16:16:30 -0500 Subject: [PATCH 065/667] Make Servers AzureCompositeResources. --- .../providers/azure/resources/sql/servers.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/servers.py b/ScoutSuite/providers/azure/resources/sql/servers.py index 48b5de91b..9eab303c2 100644 --- a/ScoutSuite/providers/azure/resources/sql/servers.py +++ b/ScoutSuite/providers/azure/resources/sql/servers.py @@ -2,7 +2,7 @@ from azure.mgmt.sql import SqlManagementClient -from ScoutSuite.providers.base.configs.resources import Resources +from ..resources import AzureCompositeResources from ScoutSuite.providers.azure.utils import get_resource_group_name from .databases import Databases @@ -10,18 +10,18 @@ from ..utils import get_non_provider_id -class Servers(Resources): +class Servers(AzureCompositeResources): children = [ Databases, ServerAzureAdAdministrators, ] - async def fetch_all(self, credentials): + # TODO: make it really async. + async def fetch_all(self, credentials, **kwargs): # sdk container: api = SqlManagementClient(credentials.credentials, credentials.subscription_id) self['servers'] = {} - # TODO: for server in await api.servers.list(): for server in api.servers.list(): id = get_non_provider_id(server.id) resource_group_name = get_resource_group_name(server.id) @@ -30,9 +30,10 @@ async def fetch_all(self, credentials): 'id': id, 'name': server.name } + await self.fetch_children( + parent=self['servers'][id], + resource_group_name=resource_group_name, + server_name=server.name, + credentials=credentials) - # put the following code in a fetch_children() parent method (typical method for a composite node)? - for resources_class in self.children: - resources = resources_class(resource_group_name, server.name) - await resources.fetch_all(credentials) - self['servers'][id].update(resources) + self['servers_count'] = len(self['servers']) From 7137f1614dbddd93f15b8278e5ba60b044acfee9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 16:16:49 -0500 Subject: [PATCH 066/667] Update services. --- ScoutSuite/providers/azure/configs/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/azure/configs/services.py b/ScoutSuite/providers/azure/configs/services.py index 793fb2ade..c5520dcfd 100644 --- a/ScoutSuite/providers/azure/configs/services.py +++ b/ScoutSuite/providers/azure/configs/services.py @@ -3,7 +3,7 @@ from ScoutSuite.providers.base.configs.services import BaseServicesConfig from ScoutSuite.providers.azure.services.storageaccounts import StorageAccountsConfig from ScoutSuite.providers.azure.services.monitor import MonitorConfig -from ScoutSuite.providers.azure.resources.sql.servers import ServersConfig as SQLDatabaseConfig +from ScoutSuite.providers.azure.resources.sql.servers import Servers as SQLDatabaseConfig from ScoutSuite.providers.azure.services.securitycenter import SecurityCenterConfig from ScoutSuite.providers.azure.services.network import NetworkConfig from ScoutSuite.providers.azure.services.keyvault import KeyVaultConfig From 1075bf072abac8d7b49a6c8a6b5e044dfa41d741 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 16:36:03 -0500 Subject: [PATCH 067/667] Use Azure API through a facade. --- .../resources/sql/database_blob_auditing_policies.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py b/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py index b2d3d74a2..1f617ca4d 100644 --- a/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py +++ b/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py @@ -7,15 +7,15 @@ class DatabaseBlobAuditingPolicies(AzureSimpleResources): - def __init__(self, resource_group_name, server_name, database_name): + def __init__(self, resource_group_name, server_name, database_name, facade): self.resource_group_name = resource_group_name self.server_name = server_name self.database_name = database_name + self.facade = facade # TODO: make it really async. - async def get_resources_from_api(self, credentials): - api = SqlManagementClient(credentials.credentials, credentials.subscription_id) - return api.database_blob_auditing_policies.get( + async def get_resources_from_api(self): + return self.facade.database_blob_auditing_policies.get( self.resource_group_name, self.server_name, self.database_name) def parse_resource(self, policies): From cf5c3b2453325335b88bad122f162229e0663e64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 16:36:26 -0500 Subject: [PATCH 068/667] Use Azure API through a facade. --- .../resources/sql/database_threat_detection_policies.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py b/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py index 2563f813a..61dc696fd 100644 --- a/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py +++ b/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py @@ -7,15 +7,15 @@ class DatabaseThreatDetectionPolicies(AzureSimpleResources): - def __init__(self, resource_group_name, server_name, database_name): + def __init__(self, resource_group_name, server_name, database_name, facade): self.resource_group_name = resource_group_name self.server_name = server_name self.database_name = database_name + self.facade = facade # TODO: make it really async. - async def get_resources_from_api(self, credentials): - api = SqlManagementClient(credentials.credentials, credentials.subscription_id) - return api.database_threat_detection_policies.get( + async def get_resources_from_api(self): + return self.facade.database_threat_detection_policies.get( self.resource_group_name, self.server_name, self.database_name) def parse_resource(self, policies): From 90f421e691c1edef982c5e2a72c824fd99391bc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 16:36:43 -0500 Subject: [PATCH 069/667] Use Azure API through a facade. --- .../providers/azure/resources/sql/replication_links.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/replication_links.py b/ScoutSuite/providers/azure/resources/sql/replication_links.py index dbedd953d..aca33b763 100644 --- a/ScoutSuite/providers/azure/resources/sql/replication_links.py +++ b/ScoutSuite/providers/azure/resources/sql/replication_links.py @@ -7,15 +7,15 @@ class ReplicationLinks(AzureSimpleResources): - def __init__(self, resource_group_name, server_name, database_name): + def __init__(self, resource_group_name, server_name, database_name, facade): self.resource_group_name = resource_group_name self.server_name = server_name self.database_name = database_name + self.facade = facade # TODO: make it really async. - async def get_resources_from_api(self, credentials): - api = SqlManagementClient(credentials.credentials, credentials.subscription_id) - return list(api.replication_links.list_by_database( + async def get_resources_from_api(self): + return list(self.facade.replication_links.list_by_database( self.resource_group_name, self.server_name, self.database_name)) def parse_resource(self, links): From fbe25108a65f34ca392fa7f547404681cbeaaea6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 16:36:56 -0500 Subject: [PATCH 070/667] Use Azure API through a facade. --- .../resources/sql/server_azure_ad_administrators.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py b/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py index a6de8eba7..1fe551488 100644 --- a/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py +++ b/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py @@ -8,17 +8,15 @@ class ServerAzureAdAdministrators(Resources): - def __init__(self, resource_group_name, server_name): + def __init__(self, resource_group_name, server_name, facade): self.resource_group_name = resource_group_name self.server_name = server_name + self.facade = facade # TODO: make it really async. - async def fetch_all(self, credentials): - # sdk container: - api = SqlManagementClient(credentials.credentials, credentials.subscription_id) - + async def fetch_all(self): try: - _ = api.server_azure_ad_administrators.get(self.resource_group_name, self.server_name) + _ = self.facade.server_azure_ad_administrators.get(self.resource_group_name, self.server_name) self['ad_admin_configured'] = True except CloudError: # no ad admin configured returns a 404 error self['ad_admin_configured'] = False From 3ab02164d146b7b386d88c8ae0d41aaf1921f06d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 16:37:07 -0500 Subject: [PATCH 071/667] Use Azure API through a facade. --- .../azure/resources/sql/transparent_data_encryptions.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py b/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py index becc2748c..e52f6e3df 100644 --- a/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py +++ b/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py @@ -7,15 +7,15 @@ class TransparentDataEncryptions(AzureSimpleResources): - def __init__(self, resource_group_name, server_name, database_name): + def __init__(self, resource_group_name, server_name, database_name, facade): self.resource_group_name = resource_group_name self.server_name = server_name self.database_name = database_name + self.facade = facade # TODO: make it really async. - async def get_resources_from_api(self, credentials): - api = SqlManagementClient(credentials.credentials, credentials.subscription_id) - return api.transparent_data_encryptions.get( + async def get_resources_from_api(self): + return self.facade.transparent_data_encryptions.get( self.resource_group_name, self.server_name, self.database_name) def parse_resource(self, encryptions): From bc9dd2f49ceff901f79944df47fd22d12a178880 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 16:37:30 -0500 Subject: [PATCH 072/667] Use Azure API through a facade and ignore master db. --- .../providers/azure/resources/sql/databases.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/databases.py b/ScoutSuite/providers/azure/resources/sql/databases.py index 8178ff3be..9a37673f7 100644 --- a/ScoutSuite/providers/azure/resources/sql/databases.py +++ b/ScoutSuite/providers/azure/resources/sql/databases.py @@ -18,16 +18,20 @@ class Databases(AzureCompositeResources): TransparentDataEncryptions, ] - def __init__(self, resource_group_name, server_name): + def __init__(self, resource_group_name, server_name, facade): self.resource_group_name = resource_group_name self.server_name = server_name + self.facade = facade # TODO: make it really async. - async def fetch_all(self, credentials): - api = SqlManagementClient(credentials.credentials, credentials.subscription_id) - + async def fetch_all(self): self['databases'] = {} - for db in api.databases.list_by_server(self.resource_group_name, self.server_name): + + for db in self.facade.databases.list_by_server(self.resource_group_name, self.server_name): + # We do not want to scan 'master' database which is auto-generated by Azure and read-only: + if db.name == 'master': + continue + self['databases'][db.name] = { 'id': db.name, } @@ -36,5 +40,5 @@ async def fetch_all(self, credentials): resource_group_name=self.resource_group_name, server_name=self.server_name, database_name=db.name, - credentials=credentials + facade=self.facade ) From cf627e31d60ff2c9f5b4565044f3b74d85ed30ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 16:39:54 -0500 Subject: [PATCH 073/667] Define a facade for using Azure API. --- ScoutSuite/providers/azure/resources/sql/servers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/servers.py b/ScoutSuite/providers/azure/resources/sql/servers.py index 9eab303c2..9c268f95e 100644 --- a/ScoutSuite/providers/azure/resources/sql/servers.py +++ b/ScoutSuite/providers/azure/resources/sql/servers.py @@ -18,11 +18,11 @@ class Servers(AzureCompositeResources): # TODO: make it really async. async def fetch_all(self, credentials, **kwargs): - # sdk container: - api = SqlManagementClient(credentials.credentials, credentials.subscription_id) + # TODO: build that facade somewhere else: + facade = SqlManagementClient(credentials.credentials, credentials.subscription_id) self['servers'] = {} - for server in api.servers.list(): + for server in facade.servers.list(): id = get_non_provider_id(server.id) resource_group_name = get_resource_group_name(server.id) @@ -34,6 +34,6 @@ async def fetch_all(self, credentials, **kwargs): parent=self['servers'][id], resource_group_name=resource_group_name, server_name=server.name, - credentials=credentials) + facade=facade) self['servers_count'] = len(self['servers']) From dcf6e3a0196de64c5aaf4ee9864cb5189c47b81c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 16:40:16 -0500 Subject: [PATCH 074/667] Remove credentials argument. --- ScoutSuite/providers/azure/resources/resources.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/resources.py b/ScoutSuite/providers/azure/resources/resources.py index 36072503e..02a8d6097 100644 --- a/ScoutSuite/providers/azure/resources/resources.py +++ b/ScoutSuite/providers/azure/resources/resources.py @@ -8,8 +8,8 @@ class AzureSimpleResources(SimpleResources, metaclass=abc.ABCMeta): - async def fetch_all(self, credentials): - raw_resource = await self.get_resources_from_api(credentials) + async def fetch_all(self): + raw_resource = await self.get_resources_from_api() name, resource = self.parse_resource(raw_resource) self[name] = resource @@ -17,8 +17,8 @@ async def fetch_all(self, credentials): class AzureCompositeResources(CompositeResources, metaclass=abc.ABCMeta): # TODO: get rid of the credentials. - async def fetch_children(self, parent, credentials, **kwargs): + async def fetch_children(self, parent, **kwargs): for child_class in self.children: child = child_class(**kwargs) - await child.fetch_all(credentials) + await child.fetch_all() parent.update(child) From d9884d702356bc957d43b43eb9f375bcb2265343 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sgha=C3=AFer?= Date: Sun, 24 Feb 2019 17:35:53 -0500 Subject: [PATCH 075/667] Fixed comment Co-Authored-By: Aboisier --- ScoutSuite/providers/aws/facade/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/utils.py b/ScoutSuite/providers/aws/facade/utils.py index 3db154e2e..f29d6f14c 100644 --- a/ScoutSuite/providers/aws/facade/utils.py +++ b/ScoutSuite/providers/aws/facade/utils.py @@ -7,9 +7,9 @@ def get_all_pages(api_call, parse_response): response = api_call() resources.extend(parse_response(response)) - # TODO: this marker should be passed to te api call. Also, some calls return a NextMarker, some return a NextToken. + # TODO: this marker should be passed to the api call. Also, some calls return a NextMarker, some return a NextToken. marker = response.get('NextMarker', None) if marker is None: break - return resources \ No newline at end of file + return resources From d547415f7ba5fd6fc8eba9c068f3ea1d6bfeaa38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sgha=C3=AFer?= Date: Sun, 24 Feb 2019 17:38:55 -0500 Subject: [PATCH 076/667] Fixed spelling mistake Co-Authored-By: Aboisier --- ScoutSuite/providers/aws/configs/regions_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/aws/configs/regions_config.py b/ScoutSuite/providers/aws/configs/regions_config.py index 5c2fedf61..95c942f4a 100644 --- a/ScoutSuite/providers/aws/configs/regions_config.py +++ b/ScoutSuite/providers/aws/configs/regions_config.py @@ -27,8 +27,8 @@ async def fetch_all(self, chosen_regions=None, partition_name='aws'): class ScopedResources(Resources): async def fetch_all(self, scope): for raw_resource in await self.get_resources_in_scope(scope): - name, ressource = self.parse_resource(raw_resource) - self[name] = ressource + name, resource = self.parse_resource(raw_resource) + self[name] = resource self.count = len(self) return self From 9d23362bd5e2b37f277034a6cf5124371f65fe9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 16:10:57 -0500 Subject: [PATCH 077/667] Add SimpleResources and CompositeResources abstract classes. --- .../providers/base/configs/resources.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/ScoutSuite/providers/base/configs/resources.py b/ScoutSuite/providers/base/configs/resources.py index d709c4c3c..c75c01766 100644 --- a/ScoutSuite/providers/base/configs/resources.py +++ b/ScoutSuite/providers/base/configs/resources.py @@ -8,3 +8,27 @@ class Resources(dict, metaclass=abc.ABCMeta): @abc.abstractmethod async def fetch_all(self, **kwargs): raise NotImplementedError() + + +class SimpleResources(Resources, metaclass=abc.ABCMeta): + + @abc.abstractmethod + def parse_resource(self, resource): + raise NotImplementedError + + @abc.abstractmethod + async def get_resources_from_api(self, **kwargs): + raise NotImplementedError + + +class CompositeResources(Resources, metaclass=abc.ABCMeta): + + # The following enforces that classes which inherits from CompositeResources define a `children` attribute: + @property + @abc.abstractmethod + def children(self): + raise NotImplementedError + + @abc.abstractmethod + async def fetch_children(self, **kwargs): + raise NotImplementedError From e2e3402e6d7d758b28d01d32c8f46074fdfb6d8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 22:41:50 -0500 Subject: [PATCH 078/667] Remove useless variable. --- .../azure/resources/sql/server_azure_ad_administrators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py b/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py index 1fe551488..778f774e1 100644 --- a/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py +++ b/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py @@ -16,7 +16,7 @@ def __init__(self, resource_group_name, server_name, facade): # TODO: make it really async. async def fetch_all(self): try: - _ = self.facade.server_azure_ad_administrators.get(self.resource_group_name, self.server_name) + self.facade.server_azure_ad_administrators.get(self.resource_group_name, self.server_name) self['ad_admin_configured'] = True except CloudError: # no ad admin configured returns a 404 error self['ad_admin_configured'] = False From f1c35dc3e9b3f4f0df3e519f29fbd4a8f1033c8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 22:42:19 -0500 Subject: [PATCH 079/667] Add docstrings. --- .../providers/base/configs/resources.py | 65 ++++++++++++++++--- 1 file changed, 57 insertions(+), 8 deletions(-) diff --git a/ScoutSuite/providers/base/configs/resources.py b/ScoutSuite/providers/base/configs/resources.py index c75c01766..44aac0412 100644 --- a/ScoutSuite/providers/base/configs/resources.py +++ b/ScoutSuite/providers/base/configs/resources.py @@ -1,34 +1,83 @@ # -*- coding: utf-8 -*- +""" +This module provides some abstract classes for representing a hierarchical structure. +Especially since all cloud providers (AWS, Azure and GCP for now) organize their resources (virtual machines, +databases, load balancers, user accounts and so on...) with some kind of hierarchy, these classes may be +used to reflect that. +""" import abc class Resources(dict, metaclass=abc.ABCMeta): + """ + This is the base class of a hierarchical structure. Everything is basically `Resources`. + It stores in its internal dictionary instances of a given type of resources, with instance ids as keys and + instance configurations (which store other nested resources) as values. + """ + @abc.abstractmethod async def fetch_all(self, **kwargs): + """Fetches, parses and stores instances of a given type of resources from a cloud provider API. + + :param kwargs: + :return: + """ raise NotImplementedError() -class SimpleResources(Resources, metaclass=abc.ABCMeta): +class CompositeResources(Resources, metaclass=abc.ABCMeta): + + """ + This class represents a node in the hierarchical structure. + As inherited from `Resources`, it still stores instances of a given type of resources internally but + also store some kind of nested resources referred as its 'children'. + """ + @property @abc.abstractmethod - def parse_resource(self, resource): + def children(self): + """A class that inherits from 'CompositeResources' should define a 'children' attribute, typically a list of + `Resources` classes. That is enforced by this abstract property. + + :return: + """ raise NotImplementedError @abc.abstractmethod - async def get_resources_from_api(self, **kwargs): + async def fetch_children(self, **kwargs): + """Fetches, parses and stores instances of nested resources included in a `CompositeResources` and defined + in the 'children' attribute. + + :param kwargs: + :return: + """ raise NotImplementedError -class CompositeResources(Resources, metaclass=abc.ABCMeta): +class SimpleResources(Resources, metaclass=abc.ABCMeta): + + """ + This class represents a leaf in the hierarchical structure. + As inherited from `Resources`, it still stores instances of a given type of resources internally. + """ - # The following enforces that classes which inherits from CompositeResources define a `children` attribute: - @property @abc.abstractmethod - def children(self): + async def get_resources_from_api(self, **kwargs): + """ Fetches resources from a cloud provider API. + + :param kwargs: + :return: + """ raise NotImplementedError @abc.abstractmethod - async def fetch_children(self, **kwargs): + def parse_resource(self, resource): + """ Parses the retrieved resources. + + :param resource: + :return: + """ raise NotImplementedError + From 2d644e6ebb7fa17581d6694b2c93f13b018359f8 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 25 Feb 2019 10:17:52 -0500 Subject: [PATCH 080/667] Implemented AWSSimpleResource and AWSCompositeResource --- .../providers/aws/resources/resources.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 ScoutSuite/providers/aws/resources/resources.py diff --git a/ScoutSuite/providers/aws/resources/resources.py b/ScoutSuite/providers/aws/resources/resources.py new file mode 100644 index 000000000..e05bb2b14 --- /dev/null +++ b/ScoutSuite/providers/aws/resources/resources.py @@ -0,0 +1,28 @@ +from ScoutSuite.providers.base.configs.resources import SimpleResources +from ScoutSuite.providers.base.configs.resources import CompositeResources +from ScoutSuite.providers.aws.facade.facade import AWSFacade +from opinel.utils.aws import get_aws_account_id +import abc + +class AWSSimpleResources(SimpleResources, metaclass=abc.ABCMeta): + def __init__(self, scope): + self.scope = scope + self.facade = AWSFacade() + + async def fetch_all(self): + raw_resources = await self.get_resources_from_api() + for raw_resource in raw_resources: + name, resource = self.parse_resource(raw_resource) + self[name] = resource + +class AWSCompositeResources(CompositeResources, metaclass=abc.ABCMeta): + + # TODO: get rid of the credentials. + async def fetch_children(self, parent, **kwargs): + for child_class, key in self.children: + child = child_class(**kwargs) + await child.fetch_all() + if parent.get(key) is None: + parent[key] = {} + parent[key].update(child) + parent[key + '_count'] = len(child) \ No newline at end of file From 9d1c05c8b0c633d5d7ec4e59c647b13629b48919 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 25 Feb 2019 10:18:12 -0500 Subject: [PATCH 081/667] Implemented Regions using AWSCompositeResource --- .../providers/aws/resources/resources.py | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/resources/resources.py b/ScoutSuite/providers/aws/resources/resources.py index e05bb2b14..20a18410c 100644 --- a/ScoutSuite/providers/aws/resources/resources.py +++ b/ScoutSuite/providers/aws/resources/resources.py @@ -25,4 +25,30 @@ async def fetch_children(self, parent, **kwargs): if parent.get(key) is None: parent[key] = {} parent[key].update(child) - parent[key + '_count'] = len(child) \ No newline at end of file + parent[key + '_count'] = len(child) + +class Regions(AWSCompositeResources, metaclass=abc.ABCMeta): + def __init__(self, service): + self.service = service + # TODO: Should be injected + self.facade = AWSFacade() + + async def fetch_all(self, credentials, chosen_regions=None, partition_name='aws'): + self['regions'] = {} + for region in await self.facade.build_region_list(self.service, chosen_regions, partition_name): + # TODO: Do we really need id, region AND name? + self['regions'][region] = { + 'id': region, + 'region': region, + 'name': region + } + + + await self.fetch_children(self['regions'][region], scope={'region': region, 'owner_id': get_aws_account_id(credentials)}) + + self._set_counts() + + def _set_counts(self): + self['regions_count'] = len(self['regions']) + for _, key in self.children: + self[key + '_count'] = sum([region[key + '_count'] for region in self['regions'].values()]) \ No newline at end of file From 46fa3fbb2f41dd0b96268f42adc3d5f86951f397 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 25 Feb 2019 10:18:48 -0500 Subject: [PATCH 082/667] Integrated AWSSimpleResources in AWSLambda service --- .../aws/services/awslambda/service.py | 48 ++++++++----------- 1 file changed, 19 insertions(+), 29 deletions(-) diff --git a/ScoutSuite/providers/aws/services/awslambda/service.py b/ScoutSuite/providers/aws/services/awslambda/service.py index 28363222a..3a2da1d5a 100644 --- a/ScoutSuite/providers/aws/services/awslambda/service.py +++ b/ScoutSuite/providers/aws/services/awslambda/service.py @@ -1,40 +1,30 @@ # -*- coding: utf-8 -*- -from ScoutSuite.providers.base.configs.resources import Resources -from ScoutSuite.providers.aws.configs.regions_config import Regions +from ScoutSuite.providers.aws.resources.resources import Regions, AWSSimpleResources from ScoutSuite.providers.aws.facade.facade import AWSFacade from opinel.utils.aws import build_region_list +class RegionalLambdas(AWSSimpleResources): + async def get_resources_from_api(self): + return self.facade.awslambda.get_functions(self.scope['region']) + + def parse_resource(self, raw_function): + raw_function['name'] = raw_function.pop('FunctionName') + return (raw_function['name'], raw_function) + + @staticmethod + def parse_function(function): + function['name'] = function.pop('FunctionName') + return (function['name'], function) + class Lambdas(Regions): + children = [ + (RegionalLambdas, 'functions') + ] + def __init__(self): super(Lambdas, self).__init__('lambda') # TODO: Remove the credentials parameter. We had to keep it for compatibility async def fetch_all(self, credentials=None, regions=None, partition_name='aws'): - await super(Lambdas, self).fetch_all( - chosen_regions=regions, - partition_name=partition_name - ) - - for region in self['regions']: - self['regions'][region] = await RegionalLambdas().fetch_all(region=region) - - -class RegionalLambdas(Resources): - async def fetch_all(self, region): - # TODO: Should be injected - facade = AWSFacade() - - functions = {} - for raw_function in facade.awslambda.get_functions(region): - name, function = self.parse_function(raw_function) - functions[name] = function - - self['functions_count'] = len(functions) - self['functions'] = functions - return self - - @staticmethod - def parse_function(function): - function['name'] = function.pop('FunctionName') - return (function['name'], function) + await super(Lambdas, self).fetch_all(credentials, regions, partition_name) \ No newline at end of file From 4ee7ad73b41cbfd73466d75db71da6b0319a2a11 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 25 Feb 2019 10:20:34 -0500 Subject: [PATCH 083/667] Integrated AWSSimpleResource to EC2Instances --- .../providers/aws/services/ec2/instances.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/ScoutSuite/providers/aws/services/ec2/instances.py b/ScoutSuite/providers/aws/services/ec2/instances.py index 52378be05..0dc469caa 100644 --- a/ScoutSuite/providers/aws/services/ec2/instances.py +++ b/ScoutSuite/providers/aws/services/ec2/instances.py @@ -1,20 +1,19 @@ -from ScoutSuite.providers.aws.configs.regions_config import ScopedResources +from ScoutSuite.providers.aws.resources.resources import AWSSimpleResources from ScoutSuite.providers.aws.facade.facade import AWSFacade from opinel.utils.aws import get_name from ScoutSuite.utils import get_keys -class EC2Instances(ScopedResources): - def __init__(self, region): - self.region = region - self.facade = AWSFacade() - +class EC2Instances(AWSSimpleResources): + async def get_resources_from_api(self): + return self.facade.ec2.get_instances(self.scope['region'], self.scope['vpc']) + def parse_resource(self, raw_instance): instance = {} id = raw_instance['InstanceId'] instance['id'] = id instance['monitoring_enabled'] = raw_instance['Monitoring']['State'] == 'enabled' - instance['user_data'] = self.facade.ec2.get_instance_user_data(self.region, id) + instance['user_data'] = self.facade.ec2.get_instance_user_data(self.scope['region'], id) # TODO: Those methods are slightly sketchy in my opinion (get methods which set stuff in a dictionary, say what) get_name(raw_instance, instance, 'InstanceId') @@ -27,6 +26,3 @@ def parse_resource(self, raw_instance): instance['network_interfaces'][eni['NetworkInterfaceId']] = nic return id, instance - - async def get_resources_in_scope(self, vpcs): - return self.facade.ec2.get_instances(self.region, vpcs) \ No newline at end of file From 515e1450436c074ba4dde3a7fe6206d487fcb381 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 25 Feb 2019 10:20:48 -0500 Subject: [PATCH 084/667] Integrated AWSSimpleResource to Snapshots --- ScoutSuite/providers/aws/services/ec2/snapshots.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/ScoutSuite/providers/aws/services/ec2/snapshots.py b/ScoutSuite/providers/aws/services/ec2/snapshots.py index d3a32755b..9ce120f68 100644 --- a/ScoutSuite/providers/aws/services/ec2/snapshots.py +++ b/ScoutSuite/providers/aws/services/ec2/snapshots.py @@ -1,17 +1,11 @@ -from ScoutSuite.providers.aws.configs.regions_config import ScopedResources +from ScoutSuite.providers.aws.resources.resources import AWSSimpleResources from ScoutSuite.providers.aws.facade.facade import AWSFacade from opinel.utils.aws import get_name -class Snapshots(ScopedResources): - - # TODO: The init could take a "scope" dictionary containing the necessary info. In this case, the owner_id and the region - def __init__(self, owner_id): - self.owner_id = owner_id - self.facade = AWSFacade() - - async def get_resources_in_scope(self, region): - return self.facade.ec2.get_snapshots(region, self.owner_id) +class Snapshots(AWSSimpleResources): + async def get_resources_from_api(self): + return self.facade.ec2.get_snapshots(self.scope['region'], self.scope['owner_id']) def parse_resource(self, raw_snapshot): raw_snapshot['id'] = raw_snapshot.pop('SnapshotId') From 5c204a0c665deddcfc9e3b18a01ce2dee8fe7ed9 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 25 Feb 2019 10:20:57 -0500 Subject: [PATCH 085/667] Integrated AWSSimpleResource to Volumes --- ScoutSuite/providers/aws/services/ec2/volumes.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/ScoutSuite/providers/aws/services/ec2/volumes.py b/ScoutSuite/providers/aws/services/ec2/volumes.py index d010ea24d..feb29bc04 100644 --- a/ScoutSuite/providers/aws/services/ec2/volumes.py +++ b/ScoutSuite/providers/aws/services/ec2/volumes.py @@ -1,16 +1,11 @@ -from ScoutSuite.providers.aws.configs.regions_config import ScopedResources +from ScoutSuite.providers.aws.resources.resources import AWSSimpleResources from ScoutSuite.providers.aws.facade.facade import AWSFacade from opinel.utils.aws import get_name -class Volumes(ScopedResources): - - # TODO: The init could take a "scope" dictionary containing the necessary info. In this case, the owner_id and the region - def __init__(self): - self.facade = AWSFacade() - - async def get_resources_in_scope(self, region): - return self.facade.ec2.get_volumes(region) +class Volumes(AWSSimpleResources): + async def get_resources_from_api(self): + return self.facade.ec2.get_volumes(self.scope['region']) def parse_resource(self, raw_volume): raw_volume['id'] = raw_volume.pop('VolumeId') From 4ea9fba2536604dcbbed5512857cb557f6109805 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Mon, 25 Feb 2019 10:21:54 -0500 Subject: [PATCH 086/667] Remove unused import statement. --- .../azure/resources/sql/database_blob_auditing_policies.py | 2 -- .../azure/resources/sql/database_threat_detection_policies.py | 2 -- ScoutSuite/providers/azure/resources/sql/databases.py | 2 -- ScoutSuite/providers/azure/resources/sql/replication_links.py | 2 -- .../azure/resources/sql/server_azure_ad_administrators.py | 1 - .../azure/resources/sql/transparent_data_encryptions.py | 2 -- 6 files changed, 11 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py b/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py index 1f617ca4d..ea2f4c116 100644 --- a/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py +++ b/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- -from azure.mgmt.sql import SqlManagementClient - from ..resources import AzureSimpleResources diff --git a/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py b/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py index 61dc696fd..4f260a3ee 100644 --- a/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py +++ b/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- -from azure.mgmt.sql import SqlManagementClient - from ..resources import AzureSimpleResources diff --git a/ScoutSuite/providers/azure/resources/sql/databases.py b/ScoutSuite/providers/azure/resources/sql/databases.py index 9a37673f7..4de894a47 100644 --- a/ScoutSuite/providers/azure/resources/sql/databases.py +++ b/ScoutSuite/providers/azure/resources/sql/databases.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- -from azure.mgmt.sql import SqlManagementClient - from ..resources import AzureCompositeResources from .database_blob_auditing_policies import DatabaseBlobAuditingPolicies diff --git a/ScoutSuite/providers/azure/resources/sql/replication_links.py b/ScoutSuite/providers/azure/resources/sql/replication_links.py index aca33b763..837988a50 100644 --- a/ScoutSuite/providers/azure/resources/sql/replication_links.py +++ b/ScoutSuite/providers/azure/resources/sql/replication_links.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- -from azure.mgmt.sql import SqlManagementClient - from ..resources import AzureSimpleResources diff --git a/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py b/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py index 778f774e1..7666757e0 100644 --- a/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py +++ b/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from azure.mgmt.sql import SqlManagementClient from msrestazure.azure_exceptions import CloudError from ScoutSuite.providers.base.configs.resources import Resources diff --git a/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py b/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py index e52f6e3df..4ebc04c89 100644 --- a/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py +++ b/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- -from azure.mgmt.sql import SqlManagementClient - from ..resources import AzureSimpleResources From 5c9f2709c992381bd868e8809e203784022066d0 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 25 Feb 2019 10:22:01 -0500 Subject: [PATCH 087/667] Integrated AWSSimpleResource to SecurityGroups --- .../aws/services/ec2/securitygroups.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/ScoutSuite/providers/aws/services/ec2/securitygroups.py b/ScoutSuite/providers/aws/services/ec2/securitygroups.py index f005ecd90..a27faceee 100644 --- a/ScoutSuite/providers/aws/services/ec2/securitygroups.py +++ b/ScoutSuite/providers/aws/services/ec2/securitygroups.py @@ -1,4 +1,4 @@ -from ScoutSuite.providers.aws.configs.regions_config import ScopedResources +from ScoutSuite.providers.aws.resources.resources import AWSSimpleResources from ScoutSuite.providers.aws.facade.facade import AWSFacade from opinel.utils.aws import get_name from ScoutSuite.utils import get_keys, ec2_classic @@ -6,16 +6,11 @@ from opinel.utils.fs import load_data -class SecurityGroups(ScopedResources): +class SecurityGroups(AWSSimpleResources): icmp_message_types_dict = load_data('icmp_message_types.json', 'icmp_message_types') - # TODO: The init could take a "scope" dictionary containing the necessary info. In this case, the owner_id and the region - def __init__(self, region): - self.region = region - self.facade = AWSFacade() - - async def get_resources_in_scope(self, vpc): - return self.facade.ec2.get_security_groups(self.region, vpc) + async def get_resources_from_api(self): + return self.facade.ec2.get_security_groups(self.scope['region'], self.scope['vpc']) def parse_resource(self, raw_security_group): security_group = {} @@ -25,18 +20,18 @@ def parse_resource(self, raw_security_group): security_group['owner_id'] = raw_security_group['OwnerId'] security_group['rules'] = {'ingress': {}, 'egress': {}} - ingress_protocols, ingress_rules_count = self.__parse_security_group_rules( + ingress_protocols, ingress_rules_count = self._parse_security_group_rules( raw_security_group['IpPermissions']) security_group['rules']['ingress']['protocols'] = ingress_protocols security_group['rules']['ingress']['count'] = ingress_rules_count - egress_protocols, egress_rules_count = self.__parse_security_group_rules( + egress_protocols, egress_rules_count = self._parse_security_group_rules( raw_security_group['IpPermissionsEgress']) security_group['rules']['egress']['protocols'] = egress_protocols security_group['rules']['egress']['count'] = egress_rules_count return security_group['id'], security_group - def __parse_security_group_rules(self, rules): + def _parse_security_group_rules(self, rules): protocols = {} rules_count = 0 for rule in rules: From 05d341685ef3fecaff17a6bc0cea2f1fa851210f Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 25 Feb 2019 10:22:20 -0500 Subject: [PATCH 088/667] Integrated AWSCompositeResource and AWSSimpleResource to Vpcs --- ScoutSuite/providers/aws/services/ec2/vpcs.py | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/ScoutSuite/providers/aws/services/ec2/vpcs.py b/ScoutSuite/providers/aws/services/ec2/vpcs.py index 48668d252..a73e28a6b 100644 --- a/ScoutSuite/providers/aws/services/ec2/vpcs.py +++ b/ScoutSuite/providers/aws/services/ec2/vpcs.py @@ -1,27 +1,32 @@ -from ScoutSuite.providers.aws.configs.regions_config import ScopedResources +from ScoutSuite.providers.aws.resources.resources import AWSCompositeResources, AWSSimpleResources from ScoutSuite.providers.aws.facade.facade import AWSFacade from ScoutSuite.providers.aws.services.ec2.instances import EC2Instances from ScoutSuite.providers.aws.services.ec2.securitygroups import SecurityGroups from ScoutSuite.providers.aws.services.ec2.networkinterfaces import NetworkInterfaces -class Vpcs(ScopedResources): - def __init__(self): +class Vpcs(AWSCompositeResources, AWSSimpleResources): + children = [ + (EC2Instances, 'instances'), + (SecurityGroups, 'security_groups'), + (NetworkInterfaces, 'network_interfaces') + ] + + def __init__(self, scope): + self.scope = scope self.facade = AWSFacade() - async def fetch_all(self, region): - await super(Vpcs, self).fetch_all(region) + async def fetch_all(self): + await super(Vpcs, self).fetch_all() for vpc in self: - # TODO: Add vpc_resource_types - self[vpc]['instances'] = await EC2Instances(region).fetch_all(vpc) - self[vpc]['security_groups'] = await SecurityGroups(region).fetch_all(vpc) - self[vpc]['network_interfaces'] = await NetworkInterfaces(region).fetch_all(vpc) + scope = {'region': self.scope['region'], 'vpc': vpc} + await self.fetch_children(self[vpc], scope=scope) return self def parse_resource(self, vpc): return vpc['VpcId'], {} - async def get_resources_in_scope(self, region): - return self.facade.ec2.get_vpcs(region) + async def get_resources_from_api(self): + return self.facade.ec2.get_vpcs(self.scope['region']) From 5d918a28571eab765d2b621b17169a70dcd76c87 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 25 Feb 2019 10:22:36 -0500 Subject: [PATCH 089/667] Integrated AWSSimpleResource to AMIs --- ScoutSuite/providers/aws/services/ec2/ami.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/ScoutSuite/providers/aws/services/ec2/ami.py b/ScoutSuite/providers/aws/services/ec2/ami.py index ba04f58e2..1fea6ef44 100644 --- a/ScoutSuite/providers/aws/services/ec2/ami.py +++ b/ScoutSuite/providers/aws/services/ec2/ami.py @@ -1,13 +1,10 @@ -from ScoutSuite.providers.aws.configs.regions_config import ScopedResources +from ScoutSuite.providers.aws.resources.resources import AWSSimpleResources from ScoutSuite.providers.aws.facade.facade import AWSFacade -class AmazonMachineImages(ScopedResources): - def __init__(self, owner_id): - self.owner_id = owner_id - self.facade = AWSFacade() - - async def get_resources_in_scope(self, region): - return self.facade.ec2.get_images(region, self.owner_id) + +class AmazonMachineImages(AWSSimpleResources): + async def get_resources_from_api(self): + return self.facade.ec2.get_images(self.scope['region'], self.scope['owner_id']) def parse_resource(self, raw_image): raw_image['id'] = raw_image['ImageId'] From af81c5b5b140c901f0c4ea08f44015efd07edc1c Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 25 Feb 2019 10:22:49 -0500 Subject: [PATCH 090/667] Integrated AWSSimpleResource to NetworkInterfaces --- .../providers/aws/services/ec2/networkinterfaces.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/ScoutSuite/providers/aws/services/ec2/networkinterfaces.py b/ScoutSuite/providers/aws/services/ec2/networkinterfaces.py index 731678260..0e8abb320 100644 --- a/ScoutSuite/providers/aws/services/ec2/networkinterfaces.py +++ b/ScoutSuite/providers/aws/services/ec2/networkinterfaces.py @@ -1,15 +1,10 @@ -from ScoutSuite.providers.aws.configs.regions_config import ScopedResources +from ScoutSuite.providers.aws.resources.resources import AWSSimpleResources from ScoutSuite.providers.aws.facade.facade import AWSFacade -class NetworkInterfaces(ScopedResources): - # TODO: The init could take a "scope" dictionary containing the necessary info. In this case, the owner_id and the region - def __init__(self, region): - self.region = region - self.facade = AWSFacade() - - async def get_resources_in_scope(self, vpc): - return self.facade.ec2.get_network_interfaces(self.region, vpc) +class NetworkInterfaces(AWSSimpleResources): + async def get_resources_from_api(self): + return self.facade.ec2.get_network_interfaces(self.scope['region'], self.scope['vpc']) def parse_resource(self, raw_network_interace): raw_network_interace['name'] = raw_network_interace['NetworkInterfaceId'] From 724177428047a79e83bd6cbf0ba5feb994612062 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 25 Feb 2019 10:22:57 -0500 Subject: [PATCH 091/667] Integrated Regions to EC2 --- .../providers/aws/services/ec2/service.py | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/ScoutSuite/providers/aws/services/ec2/service.py b/ScoutSuite/providers/aws/services/ec2/service.py index 018ab6aca..bbbf36aea 100644 --- a/ScoutSuite/providers/aws/services/ec2/service.py +++ b/ScoutSuite/providers/aws/services/ec2/service.py @@ -1,5 +1,4 @@ -from ScoutSuite.providers.base.configs.resources import Resources -from ScoutSuite.providers.aws.configs.regions_config import Regions, ScopedResources +from ScoutSuite.providers.aws.resources.resources import Regions from ScoutSuite.providers.aws.services.ec2.ami import AmazonMachineImages from ScoutSuite.providers.aws.services.ec2.vpcs import Vpcs from ScoutSuite.providers.aws.services.ec2.snapshots import Snapshots @@ -12,24 +11,25 @@ # TODO: Add docstrings class EC2(Regions): + children = [ + (Vpcs, 'vpcs'), + (AmazonMachineImages, 'images'), + (Snapshots, 'snapshots'), + (Volumes, 'volumes') + ] + def __init__(self): super(EC2, self).__init__('ec2') async def fetch_all(self, credentials=None, regions=None, partition_name='aws'): - await super(EC2, self).fetch_all(chosen_regions=regions, partition_name=partition_name) + await super(EC2, self).fetch_all(credentials, regions, partition_name) - # TODO: Is there a way to generalize this? for region in self['regions']: - self['regions'][region]['vpcs'] = await Vpcs().fetch_all(region) - self['regions'][region]['instances_count'] = sum([vpc['instances'].count for vpc in self['regions'][region]['vpcs'].values()]) - self['regions'][region]['security_groups_count'] = sum([vpc['security_groups'].count for vpc in self['regions'][region]['vpcs'].values()]) - self['regions'][region]['network_interfaces_count'] = sum([vpc['network_interfaces'].count for vpc in self['regions'][region]['vpcs'].values()]) - - self['regions'][region]['images'] = await AmazonMachineImages(get_aws_account_id(credentials)).fetch_all(region) - self['regions'][region]['images_count'] = self['regions'][region]['images'].count - - self['regions'][region]['snapshots'] = await Snapshots(get_aws_account_id(credentials)).fetch_all(region) - self['regions'][region]['snapshots_count'] = self['regions'][region]['snapshots'].count - - self['regions'][region]['volumes'] = await Volumes().fetch_all(region) - self['regions'][region]['volumes_count'] = self['regions'][region]['volumes'].count \ No newline at end of file + # TODO: Is there a way to move this elsewhere? + self['regions'][region]['instances_count'] = sum([len(vpc['instances']) for vpc in self['regions'][region]['vpcs'].values()]) + self['regions'][region]['security_groups_count'] = sum([len(vpc['security_groups']) for vpc in self['regions'][region]['vpcs'].values()]) + self['regions'][region]['network_interfaces_count'] = sum([len(vpc['network_interfaces']) for vpc in self['regions'][region]['vpcs'].values()]) + + self['instances_count'] = sum([region['instances_count'] for region in self['regions'].values()]) + self['security_groups_count'] = sum([region['security_groups_count'] for region in self['regions'].values()]) + self['network_interfaces_count'] = sum([region['network_interfaces_count'] for region in self['regions'].values()]) From 56a5114c7e7d6a432588f565d961a3594955df88 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 25 Feb 2019 10:23:07 -0500 Subject: [PATCH 092/667] Got rid of ScopedResources and the old Regions implementation --- .../providers/aws/configs/regions_config.py | 42 ------------------- 1 file changed, 42 deletions(-) delete mode 100644 ScoutSuite/providers/aws/configs/regions_config.py diff --git a/ScoutSuite/providers/aws/configs/regions_config.py b/ScoutSuite/providers/aws/configs/regions_config.py deleted file mode 100644 index 95c942f4a..000000000 --- a/ScoutSuite/providers/aws/configs/regions_config.py +++ /dev/null @@ -1,42 +0,0 @@ -# -*- coding: utf-8 -*- -from ScoutSuite.providers.base.configs.resources import Resources -from ScoutSuite.providers.aws.facade.facade import AWSFacade -import abc - - -class Regions(Resources): - def __init__(self, service): - self._service = service - - async def fetch_all(self, chosen_regions=None, partition_name='aws'): - # TODO: Should be injected - facade = AWSFacade() - - self['regions'] = {} - for region in await facade.build_region_list(self._service, chosen_regions, partition_name): - # TODO: Do we really need id, region AND name? - self['regions'][region] = { - 'id': region, - 'region': region, - 'name': region - } - - self['regions_count'] = len(self['regions']) - - -class ScopedResources(Resources): - async def fetch_all(self, scope): - for raw_resource in await self.get_resources_in_scope(scope): - name, resource = self.parse_resource(raw_resource) - self[name] = resource - - self.count = len(self) - return self - - @abc.abstractclassmethod - def parse_resource(self, resource): - raise NotImplementedError() - - @abc.abstractclassmethod - async def get_resources_in_scope(self, region): - raise NotImplementedError() From 3c8e0df873aca422f90f65686a23f1bdb60125a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Mon, 25 Feb 2019 14:25:35 -0500 Subject: [PATCH 093/667] Make parse_resource method update its internal dict instead of returning a tuple. --- ScoutSuite/providers/azure/resources/resources.py | 3 +-- .../azure/resources/sql/database_blob_auditing_policies.py | 4 +++- .../azure/resources/sql/database_threat_detection_policies.py | 4 +++- ScoutSuite/providers/azure/resources/sql/replication_links.py | 4 +++- .../azure/resources/sql/transparent_data_encryptions.py | 4 +++- 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/resources.py b/ScoutSuite/providers/azure/resources/resources.py index 02a8d6097..2749ec9ff 100644 --- a/ScoutSuite/providers/azure/resources/resources.py +++ b/ScoutSuite/providers/azure/resources/resources.py @@ -10,8 +10,7 @@ class AzureSimpleResources(SimpleResources, metaclass=abc.ABCMeta): async def fetch_all(self): raw_resource = await self.get_resources_from_api() - name, resource = self.parse_resource(raw_resource) - self[name] = resource + self.parse_resource(raw_resource) class AzureCompositeResources(CompositeResources, metaclass=abc.ABCMeta): diff --git a/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py b/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py index ea2f4c116..637eae5dd 100644 --- a/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py +++ b/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py @@ -17,4 +17,6 @@ async def get_resources_from_api(self): self.resource_group_name, self.server_name, self.database_name) def parse_resource(self, policies): - return 'auditing_enabled', policies.state == "Enabled" + self.update({ + 'auditing_enabled': policies.state == "Enabled" + }) diff --git a/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py b/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py index 4f260a3ee..84977378f 100644 --- a/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py +++ b/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py @@ -17,4 +17,6 @@ async def get_resources_from_api(self): self.resource_group_name, self.server_name, self.database_name) def parse_resource(self, policies): - return 'threat_detection_enabled', policies.state == "Enabled" + self.update({ + 'threat_detection_enabled': policies.state == "Enabled" + }) diff --git a/ScoutSuite/providers/azure/resources/sql/replication_links.py b/ScoutSuite/providers/azure/resources/sql/replication_links.py index 837988a50..f15e7e27c 100644 --- a/ScoutSuite/providers/azure/resources/sql/replication_links.py +++ b/ScoutSuite/providers/azure/resources/sql/replication_links.py @@ -17,4 +17,6 @@ async def get_resources_from_api(self): self.resource_group_name, self.server_name, self.database_name)) def parse_resource(self, links): - return 'replication_configured', len(links) > 0 + self.update({ + 'replication_configured': len(links) > 0 + }) diff --git a/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py b/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py index 4ebc04c89..d82b51fce 100644 --- a/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py +++ b/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py @@ -17,4 +17,6 @@ async def get_resources_from_api(self): self.resource_group_name, self.server_name, self.database_name) def parse_resource(self, encryptions): - return 'transparent_data_encryption_enabled', encryptions.status == "Enabled" + return self.update({ + 'transparent_data_encryption_enabled': encryptions.status == "Enabled" + }) From 8248fd51f4f877992af1dd5844148c26d7b519e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Mon, 25 Feb 2019 10:40:12 -0500 Subject: [PATCH 094/667] Parse auditing config of SQL servers. --- .../sql/server_blob_auditing_policies.py | 19 +++++++++++++++++++ .../providers/azure/resources/sql/servers.py | 2 ++ 2 files changed, 21 insertions(+) create mode 100644 ScoutSuite/providers/azure/resources/sql/server_blob_auditing_policies.py diff --git a/ScoutSuite/providers/azure/resources/sql/server_blob_auditing_policies.py b/ScoutSuite/providers/azure/resources/sql/server_blob_auditing_policies.py new file mode 100644 index 000000000..18d7a7dda --- /dev/null +++ b/ScoutSuite/providers/azure/resources/sql/server_blob_auditing_policies.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- + +from ..resources import AzureSimpleResources + + +class ServerBlobAuditingPolicies(AzureSimpleResources): + + def __init__(self, resource_group_name, server_name, facade): + self.resource_group_name = resource_group_name + self.server_name = server_name + self.facade = facade + + # TODO: make it really async. + async def get_resources_from_api(self): + return self.facade.server_blob_auditing_policies.get( + self.resource_group_name, self.server_name) + + def parse_resource(self, policies): + return 'auditing_enabled', policies.state == "Enabled" diff --git a/ScoutSuite/providers/azure/resources/sql/servers.py b/ScoutSuite/providers/azure/resources/sql/servers.py index 9c268f95e..b9635db1d 100644 --- a/ScoutSuite/providers/azure/resources/sql/servers.py +++ b/ScoutSuite/providers/azure/resources/sql/servers.py @@ -7,6 +7,7 @@ from .databases import Databases from .server_azure_ad_administrators import ServerAzureAdAdministrators +from .server_blob_auditing_policies import ServerBlobAuditingPolicies from ..utils import get_non_provider_id @@ -14,6 +15,7 @@ class Servers(AzureCompositeResources): children = [ Databases, ServerAzureAdAdministrators, + ServerBlobAuditingPolicies ] # TODO: make it really async. From 3cea10a30cfd09317be08e5d6e91c6d6a58abf89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Mon, 25 Feb 2019 10:40:46 -0500 Subject: [PATCH 095/667] Add a new rule to check whether auditing have been enabled for all SQL servers. --- .../findings/sqldatabase-servers-no-auditing.json | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 ScoutSuite/providers/azure/rules/findings/sqldatabase-servers-no-auditing.json diff --git a/ScoutSuite/providers/azure/rules/findings/sqldatabase-servers-no-auditing.json b/ScoutSuite/providers/azure/rules/findings/sqldatabase-servers-no-auditing.json new file mode 100644 index 000000000..1c8d6bcaa --- /dev/null +++ b/ScoutSuite/providers/azure/rules/findings/sqldatabase-servers-no-auditing.json @@ -0,0 +1,11 @@ +{ + "dashboard_name": "SQL Servers", + "description": "Auditing disabled on SQL servers", + "rationale": "You should enable auditing for all of your SQL servers (CIS 4.1.1).", + "path": "sqldatabase.servers.id", + "display_path": "sqldatabase.servers.id", + "conditions": [ "and", + [ "sqldatabase.servers.id.auditing_enabled", "false", "" ] + ], + "id_suffix": "auditing_enabled" +} \ No newline at end of file From 38f1b87ec63dc6279261e8fcb8f9765934a1e6f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Mon, 25 Feb 2019 10:41:16 -0500 Subject: [PATCH 096/667] Add the new rule to the default ruleset. --- ScoutSuite/providers/azure/rules/rulesets/default.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ScoutSuite/providers/azure/rules/rulesets/default.json b/ScoutSuite/providers/azure/rules/rulesets/default.json index c79203c49..a3102b687 100644 --- a/ScoutSuite/providers/azure/rules/rulesets/default.json +++ b/ScoutSuite/providers/azure/rules/rulesets/default.json @@ -84,6 +84,12 @@ "level": "warning" } ], + "sqldatabase-servers-no-auditing.json": [ + { + "enabled": true, + "level": "warning" + } + ], "securitycenter-standard-tier-not-enabled.json": [ { "enabled": true, From 8814367d5aeed0956f975bcd90c0be4f7c12da70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Mon, 25 Feb 2019 10:41:39 -0500 Subject: [PATCH 097/667] Update HTML template for the new finding. --- .../data/html/partials/azure/services.sqldatabase.servers.html | 1 + 1 file changed, 1 insertion(+) diff --git a/ScoutSuite/output/data/html/partials/azure/services.sqldatabase.servers.html b/ScoutSuite/output/data/html/partials/azure/services.sqldatabase.servers.html index 2f7d8a2c1..1f277b709 100644 --- a/ScoutSuite/output/data/html/partials/azure/services.sqldatabase.servers.html +++ b/ScoutSuite/output/data/html/partials/azure/services.sqldatabase.servers.html @@ -8,6 +8,7 @@

    {{name}}

    Information

    SQL Server Name: {{name}}
    Azure Active Directory admin configured: {{ad_admin_configured}}
    +
    Auditing: {{ convert_bool_to_enabled auditing_enabled }}
    From 48e9ceb906475a17774f31e1586ed02cd69711d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Mon, 25 Feb 2019 10:41:58 -0500 Subject: [PATCH 098/667] Update rule description. --- .../azure/rules/findings/sqldatabase-databases-no-auditing.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-no-auditing.json b/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-no-auditing.json index 56071b778..4573ac09c 100644 --- a/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-no-auditing.json +++ b/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-no-auditing.json @@ -1,6 +1,6 @@ { "dashboard_name": "SQL Databases", - "description": "Auditing disabled", + "description": "Auditing disabled on SQL databases", "rationale": "You should enable auditing for all of your SQL databases. See CIS 4.2.1.", "path": "sqldatabase.servers.id.databases.id", "display_path": "sqldatabase.servers.id", From 2ea10d30aa918256c364ae721a70b4031499615c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Mon, 25 Feb 2019 11:19:32 -0500 Subject: [PATCH 099/667] Update rule rationale. --- .../azure/rules/findings/sqldatabase-databases-no-auditing.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-no-auditing.json b/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-no-auditing.json index 4573ac09c..55fd65d10 100644 --- a/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-no-auditing.json +++ b/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-no-auditing.json @@ -1,7 +1,7 @@ { "dashboard_name": "SQL Databases", "description": "Auditing disabled on SQL databases", - "rationale": "You should enable auditing for all of your SQL databases. See CIS 4.2.1.", + "rationale": "You should enable auditing for all of your SQL databases (CIS 4.2.1).", "path": "sqldatabase.servers.id.databases.id", "display_path": "sqldatabase.servers.id", "conditions": [ "and", From 550b14dc3d93bf23b77ea65533c1d1e9c7135785 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Mon, 25 Feb 2019 14:29:34 -0500 Subject: [PATCH 100/667] Update parse_resource method. --- .../azure/resources/sql/server_blob_auditing_policies.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/providers/azure/resources/sql/server_blob_auditing_policies.py b/ScoutSuite/providers/azure/resources/sql/server_blob_auditing_policies.py index 18d7a7dda..606f00c8d 100644 --- a/ScoutSuite/providers/azure/resources/sql/server_blob_auditing_policies.py +++ b/ScoutSuite/providers/azure/resources/sql/server_blob_auditing_policies.py @@ -16,4 +16,6 @@ async def get_resources_from_api(self): self.resource_group_name, self.server_name) def parse_resource(self, policies): - return 'auditing_enabled', policies.state == "Enabled" + self.update({ + 'auditing_enabled': policies.state == "Enabled" + }) From 6c7cd66a2ba1d2c80f60a0abeef03efca11eee35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sgha=C3=AFer?= Date: Mon, 25 Feb 2019 22:27:03 -0500 Subject: [PATCH 101/667] Update ScoutSuite/providers/aws/resources/resources.py Co-Authored-By: Aboisier --- ScoutSuite/providers/aws/resources/resources.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/resources.py b/ScoutSuite/providers/aws/resources/resources.py index 20a18410c..89597f79d 100644 --- a/ScoutSuite/providers/aws/resources/resources.py +++ b/ScoutSuite/providers/aws/resources/resources.py @@ -43,7 +43,6 @@ async def fetch_all(self, credentials, chosen_regions=None, partition_name='aws' 'name': region } - await self.fetch_children(self['regions'][region], scope={'region': region, 'owner_id': get_aws_account_id(credentials)}) self._set_counts() @@ -51,4 +50,4 @@ async def fetch_all(self, credentials, chosen_regions=None, partition_name='aws' def _set_counts(self): self['regions_count'] = len(self['regions']) for _, key in self.children: - self[key + '_count'] = sum([region[key + '_count'] for region in self['regions'].values()]) \ No newline at end of file + self[key + '_count'] = sum([region[key + '_count'] for region in self['regions'].values()]) From 32c3ad6073958cb6bbb9aff850e314e79908fa7a Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 27 Feb 2019 22:59:19 -0500 Subject: [PATCH 102/667] Fixed missing async --- ScoutSuite/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/__main__.py b/ScoutSuite/__main__.py index 53e378d12..c4e129933 100644 --- a/ScoutSuite/__main__.py +++ b/ScoutSuite/__main__.py @@ -19,7 +19,7 @@ # noinspection PyBroadException -def main(args=None): +async def main(args=None): """ Main method that runs a scan From 203145b35264b912ae12a326844d72de7b1ce9d2 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 27 Feb 2019 23:19:42 -0500 Subject: [PATCH 103/667] Removed comments --- ScoutSuite/providers/aws/resources/resources.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/resources.py b/ScoutSuite/providers/aws/resources/resources.py index 89597f79d..a26d534ac 100644 --- a/ScoutSuite/providers/aws/resources/resources.py +++ b/ScoutSuite/providers/aws/resources/resources.py @@ -16,8 +16,6 @@ async def fetch_all(self): self[name] = resource class AWSCompositeResources(CompositeResources, metaclass=abc.ABCMeta): - - # TODO: get rid of the credentials. async def fetch_children(self, parent, **kwargs): for child_class, key in self.children: child = child_class(**kwargs) @@ -36,7 +34,6 @@ def __init__(self, service): async def fetch_all(self, credentials, chosen_regions=None, partition_name='aws'): self['regions'] = {} for region in await self.facade.build_region_list(self.service, chosen_regions, partition_name): - # TODO: Do we really need id, region AND name? self['regions'][region] = { 'id': region, 'region': region, From f74d5ee6d114b4c051d7a4561577e9c3de00474e Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 27 Feb 2019 23:19:58 -0500 Subject: [PATCH 104/667] Removed unused imports --- ScoutSuite/providers/aws/services/ec2/service.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/ScoutSuite/providers/aws/services/ec2/service.py b/ScoutSuite/providers/aws/services/ec2/service.py index bbbf36aea..fa38f80f9 100644 --- a/ScoutSuite/providers/aws/services/ec2/service.py +++ b/ScoutSuite/providers/aws/services/ec2/service.py @@ -4,9 +4,6 @@ from ScoutSuite.providers.aws.services.ec2.snapshots import Snapshots from ScoutSuite.providers.aws.services.ec2.volumes import Volumes -from opinel.utils.aws import get_aws_account_id -from ScoutSuite.utils import get_keys, ec2_classic - # TODO: Add docstrings From 5809e2f6af1171c1009d31699bfbc05ab0febcd1 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 27 Feb 2019 23:20:07 -0500 Subject: [PATCH 105/667] Fixed imports --- ScoutSuite/providers/aws/services/ec2/instances.py | 2 +- ScoutSuite/providers/aws/services/ec2/securitygroups.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/aws/services/ec2/instances.py b/ScoutSuite/providers/aws/services/ec2/instances.py index 0dc469caa..1b414e9a4 100644 --- a/ScoutSuite/providers/aws/services/ec2/instances.py +++ b/ScoutSuite/providers/aws/services/ec2/instances.py @@ -1,7 +1,7 @@ from ScoutSuite.providers.aws.resources.resources import AWSSimpleResources from ScoutSuite.providers.aws.facade.facade import AWSFacade from opinel.utils.aws import get_name -from ScoutSuite.utils import get_keys +from ScoutSuite.providers.aws.utils import ec2_classic, get_keys class EC2Instances(AWSSimpleResources): diff --git a/ScoutSuite/providers/aws/services/ec2/securitygroups.py b/ScoutSuite/providers/aws/services/ec2/securitygroups.py index a27faceee..b423413da 100644 --- a/ScoutSuite/providers/aws/services/ec2/securitygroups.py +++ b/ScoutSuite/providers/aws/services/ec2/securitygroups.py @@ -1,7 +1,7 @@ from ScoutSuite.providers.aws.resources.resources import AWSSimpleResources from ScoutSuite.providers.aws.facade.facade import AWSFacade from opinel.utils.aws import get_name -from ScoutSuite.utils import get_keys, ec2_classic +from ScoutSuite.providers.aws.utils import ec2_classic, get_keys from opinel.utils.globals import manage_dictionary from opinel.utils.fs import load_data From 4c7239838431abf570e54b1436000848ed464cd1 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 27 Feb 2019 23:20:14 -0500 Subject: [PATCH 106/667] Cleaned up method --- ScoutSuite/providers/aws/services/awslambda/service.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/ScoutSuite/providers/aws/services/awslambda/service.py b/ScoutSuite/providers/aws/services/awslambda/service.py index 3a2da1d5a..5e2f41307 100644 --- a/ScoutSuite/providers/aws/services/awslambda/service.py +++ b/ScoutSuite/providers/aws/services/awslambda/service.py @@ -3,19 +3,15 @@ from ScoutSuite.providers.aws.facade.facade import AWSFacade from opinel.utils.aws import build_region_list + class RegionalLambdas(AWSSimpleResources): async def get_resources_from_api(self): return self.facade.awslambda.get_functions(self.scope['region']) - + def parse_resource(self, raw_function): raw_function['name'] = raw_function.pop('FunctionName') return (raw_function['name'], raw_function) - @staticmethod - def parse_function(function): - function['name'] = function.pop('FunctionName') - return (function['name'], function) - class Lambdas(Regions): children = [ @@ -27,4 +23,4 @@ def __init__(self): # TODO: Remove the credentials parameter. We had to keep it for compatibility async def fetch_all(self, credentials=None, regions=None, partition_name='aws'): - await super(Lambdas, self).fetch_all(credentials, regions, partition_name) \ No newline at end of file + await super(Lambdas, self).fetch_all(credentials, regions, partition_name) From 46b41cfb5627c7809e18c15abf3f62782e8b53e1 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Thu, 28 Feb 2019 08:43:47 -0500 Subject: [PATCH 107/667] Removed references to opinel --- ScoutSuite/providers/aws/resources/resources.py | 2 +- ScoutSuite/providers/aws/services/awslambda/service.py | 2 +- ScoutSuite/providers/aws/services/ec2/instances.py | 2 +- ScoutSuite/providers/aws/services/ec2/securitygroups.py | 9 +++++---- ScoutSuite/providers/aws/services/ec2/snapshots.py | 2 +- ScoutSuite/providers/aws/services/ec2/volumes.py | 2 +- requirements.txt | 5 ++--- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/resources.py b/ScoutSuite/providers/aws/resources/resources.py index a26d534ac..03fb85df3 100644 --- a/ScoutSuite/providers/aws/resources/resources.py +++ b/ScoutSuite/providers/aws/resources/resources.py @@ -1,7 +1,7 @@ from ScoutSuite.providers.base.configs.resources import SimpleResources from ScoutSuite.providers.base.configs.resources import CompositeResources from ScoutSuite.providers.aws.facade.facade import AWSFacade -from opinel.utils.aws import get_aws_account_id +from ScoutSuite.providers.aws.aws import get_aws_account_id import abc class AWSSimpleResources(SimpleResources, metaclass=abc.ABCMeta): diff --git a/ScoutSuite/providers/aws/services/awslambda/service.py b/ScoutSuite/providers/aws/services/awslambda/service.py index 5e2f41307..d7d69e7c8 100644 --- a/ScoutSuite/providers/aws/services/awslambda/service.py +++ b/ScoutSuite/providers/aws/services/awslambda/service.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from ScoutSuite.providers.aws.resources.resources import Regions, AWSSimpleResources from ScoutSuite.providers.aws.facade.facade import AWSFacade -from opinel.utils.aws import build_region_list +from ScoutSuite.providers.aws.aws import build_region_list class RegionalLambdas(AWSSimpleResources): diff --git a/ScoutSuite/providers/aws/services/ec2/instances.py b/ScoutSuite/providers/aws/services/ec2/instances.py index 1b414e9a4..3217abacd 100644 --- a/ScoutSuite/providers/aws/services/ec2/instances.py +++ b/ScoutSuite/providers/aws/services/ec2/instances.py @@ -1,6 +1,6 @@ from ScoutSuite.providers.aws.resources.resources import AWSSimpleResources from ScoutSuite.providers.aws.facade.facade import AWSFacade -from opinel.utils.aws import get_name +from ScoutSuite.providers.aws.aws import get_name from ScoutSuite.providers.aws.utils import ec2_classic, get_keys diff --git a/ScoutSuite/providers/aws/services/ec2/securitygroups.py b/ScoutSuite/providers/aws/services/ec2/securitygroups.py index b423413da..ee6edeabc 100644 --- a/ScoutSuite/providers/aws/services/ec2/securitygroups.py +++ b/ScoutSuite/providers/aws/services/ec2/securitygroups.py @@ -1,13 +1,14 @@ from ScoutSuite.providers.aws.resources.resources import AWSSimpleResources from ScoutSuite.providers.aws.facade.facade import AWSFacade -from opinel.utils.aws import get_name +from ScoutSuite.providers.aws.aws import get_name from ScoutSuite.providers.aws.utils import ec2_classic, get_keys -from opinel.utils.globals import manage_dictionary -from opinel.utils.fs import load_data +from ScoutSuite.utils import manage_dictionary +from ScoutSuite.core.fs import load_data class SecurityGroups(AWSSimpleResources): - icmp_message_types_dict = load_data('icmp_message_types.json', 'icmp_message_types') + icmp_message_types_dict = load_data( + 'icmp_message_types.json', 'icmp_message_types') async def get_resources_from_api(self): return self.facade.ec2.get_security_groups(self.scope['region'], self.scope['vpc']) diff --git a/ScoutSuite/providers/aws/services/ec2/snapshots.py b/ScoutSuite/providers/aws/services/ec2/snapshots.py index 9ce120f68..b589a91f1 100644 --- a/ScoutSuite/providers/aws/services/ec2/snapshots.py +++ b/ScoutSuite/providers/aws/services/ec2/snapshots.py @@ -1,6 +1,6 @@ from ScoutSuite.providers.aws.resources.resources import AWSSimpleResources from ScoutSuite.providers.aws.facade.facade import AWSFacade -from opinel.utils.aws import get_name +from ScoutSuite.providers.aws.aws import get_name class Snapshots(AWSSimpleResources): diff --git a/ScoutSuite/providers/aws/services/ec2/volumes.py b/ScoutSuite/providers/aws/services/ec2/volumes.py index feb29bc04..4b3017239 100644 --- a/ScoutSuite/providers/aws/services/ec2/volumes.py +++ b/ScoutSuite/providers/aws/services/ec2/volumes.py @@ -1,6 +1,6 @@ from ScoutSuite.providers.aws.resources.resources import AWSSimpleResources from ScoutSuite.providers.aws.facade.facade import AWSFacade -from opinel.utils.aws import get_name +from ScoutSuite.providers.aws.aws import get_name class Volumes(AWSSimpleResources): diff --git a/requirements.txt b/requirements.txt index 4fd46b27c..bf3eb4312 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,13 @@ # Core python-dateutil>=2.2 netaddr>=0.7.11 +iampoliciesgonewild>=1.0.6.2 # AWS Provider boto3>=1.9.60 botocore>=1.12.60 - -# Opinel requests>=2.4.0,<3.0.0 -iampoliciesgonewild>=1.0.6.2 + # GCP Provider grpcio>=1.18.0 From 84f48096dc7bd8d9d397fed8414d4438059d4269 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Thu, 28 Feb 2019 09:39:59 -0500 Subject: [PATCH 108/667] Merge refactoring/resource-configs into refactoring/aws/ec2 --- Scout.py | 1 + ScoutSuite/__main__.py | 64 ++++++++++++++++--------- ScoutSuite/providers/aws/credentials.py | 16 ++----- ScoutSuite/providers/aws/provider.py | 6 +-- 4 files changed, 50 insertions(+), 37 deletions(-) diff --git a/Scout.py b/Scout.py index 45ef3720f..83cb235dd 100755 --- a/Scout.py +++ b/Scout.py @@ -10,3 +10,4 @@ loop = asyncio.get_event_loop() loop.run_until_complete(main()) loop.close() + sys.exit() diff --git a/ScoutSuite/__main__.py b/ScoutSuite/__main__.py index c4e129933..de4785a07 100644 --- a/ScoutSuite/__main__.py +++ b/ScoutSuite/__main__.py @@ -37,7 +37,8 @@ async def main(args=None): # Create a cloud provider object cloud_provider = get_provider(provider=args.get('provider'), - profile=args.get('profile'), + profile=args.get('profile')[0] if args.get( + 'profile') else None, project_id=args.get('project_id'), folder_id=args.get('folder_id'), organization_id=args.get('organization_id'), @@ -45,7 +46,8 @@ async def main(args=None): report_dir=args.get('report_dir'), timestamp=args.get('timestamp'), services=args.get('services'), - skipped_services=args.get('skipped_services'), + skipped_services=args.get( + 'skipped_services'), thread_config=args.get('thread_config'), ) @@ -54,27 +56,41 @@ async def main(args=None): # TODO move this to after authentication, so that the report can be more specific to what's being scanned. # For example if scanning with a GCP service account, the SA email can only be known after authenticating... # Create a new report - report = Scout2Report(args.get('provider'), report_file_name, args.get('report_dir'), args.get('timestamp')) + report = Scout2Report(args.get('provider'), report_file_name, args.get( + 'report_dir'), args.get('timestamp')) # Complete run, including pulling data from provider if not args.get('fetch_local'): # Authenticate to the cloud provider - authenticated = cloud_provider.authenticate(profile=args.get('profile'), - csv_credentials=args.get('csv_credentials'), - mfa_serial=args.get('mfa_serial'), - mfa_code=args.get('mfa_code'), - user_account=args.get('user_account'), - service_account=args.get('service_account'), + authenticated = cloud_provider.authenticate(profile=args.get('profile')[0] if args.get('profile') else None, + csv_credentials=args.get( + 'csv_credentials'), + mfa_serial=args.get( + 'mfa_serial'), + mfa_code=args.get( + 'mfa_code'), + user_account=args.get( + 'user_account'), + service_account=args.get( + 'service_account'), cli=args.get('cli'), msi=args.get('msi'), - service_principal=args.get('service_principal'), - file_auth=args.get('file_auth'), - tenant_id=args.get('tenant_id'), - subscription_id=args.get('subscription_id'), - client_id=args.get('client_id'), - client_secret=args.get('client_secret'), - username=args.get('username'), - password=args.get('password') + service_principal=args.get( + 'service_principal'), + file_auth=args.get( + 'file_auth'), + tenant_id=args.get( + 'tenant_id'), + subscription_id=args.get( + 'subscription_id'), + client_id=args.get( + 'client_id'), + client_secret=args.get( + 'client_secret'), + username=args.get( + 'username'), + password=args.get( + 'password') ) if not authenticated: @@ -103,10 +119,11 @@ async def main(args=None): setattr(cloud_provider, key, last_run_dict[key]) # Pre processing - cloud_provider.preprocessing(args.get('ip_ranges'), args.get('ip_ranges_name_key')) + cloud_provider.preprocessing( + args.get('ip_ranges'), args.get('ip_ranges_name_key')) # Analyze config - finding_rules = Ruleset(environment_name=args.get('profile'), + finding_rules = Ruleset(environment_name=args.get('profile')[0] if args.get('profile') else None, cloud_provider=args.get('provider'), filename=args.get('ruleset'), ip_ranges=args.get('ip_ranges'), @@ -124,11 +141,13 @@ async def main(args=None): # Handle exceptions try: - exceptions = RuleExceptions(args.get('profile'), args.get('exceptions')[0]) + exceptions = RuleExceptions(args.get('profile')[0] if args.get( + 'profile') else None, args.get('exceptions')[0]) exceptions.process(cloud_provider) exceptions = exceptions.exceptions except Exception as e: - print_debug('Warning, failed to load exceptions. The file may not exist or may have an invalid format.') + print_debug( + 'Warning, failed to load exceptions. The file may not exist or may have an invalid format.') exceptions = {} # Finalize @@ -153,7 +172,8 @@ async def main(args=None): pass # Save config and create HTML report - html_report_path = report.save(cloud_provider, exceptions, args.get('force_write'), args.get('debug')) + html_report_path = report.save( + cloud_provider, exceptions, args.get('force_write'), args.get('debug')) # Open the report by default if not args.get('no_browser'): diff --git a/ScoutSuite/providers/aws/credentials.py b/ScoutSuite/providers/aws/credentials.py index 0bd2eb8ce..3ce700dcb 100644 --- a/ScoutSuite/providers/aws/credentials.py +++ b/ScoutSuite/providers/aws/credentials.py @@ -53,14 +53,13 @@ ######################################## -def assume_role(role_name, credentials, role_arn, role_session_name, silent=False): +def assume_role(role_name, credentials, role_arn, silent=False): """ Assume role and save credentials :param role_name: :param credentials: :param role_arn: - :param role_session_name: :param silent: :return: """ @@ -68,10 +67,7 @@ def assume_role(role_name, credentials, role_arn, role_session_name, silent=Fals # Connect to STS sts_client = connect_service('sts', credentials, silent=silent) # Set required arguments for assume role call - sts_args = { - 'RoleArn': role_arn, - 'RoleSessionName': role_session_name - } + sts_args = {'RoleArn': role_arn} # MFA used ? if 'mfa_serial' in credentials and 'mfa_code' in credentials: sts_args['TokenCode'] = credentials['mfa_code'] @@ -83,8 +79,6 @@ def assume_role(role_name, credentials, role_arn, role_session_name, silent=Fals sts_response = sts_client.assume_role(**sts_args) credentials = sts_response['Credentials'] cached_credentials_filename = get_cached_credentials_filename(role_name, role_arn) - # with open(cached_credentials_filename, 'wt+') as f: - # write_data_to_file(f, sts_response, True, False) cached_credentials_path = os.path.dirname(cached_credentials_filename) if not os.path.isdir(cached_credentials_path): os.makedirs(cached_credentials_path) @@ -432,8 +426,7 @@ def complete_profile(f, credentials, session_token_written, mfa_serial_written): # noinspection PyBroadException,PyBroadException -def read_creds(profile_name, csv_file=None, mfa_serial_arg=None, mfa_code=None, force_init=False, - role_session_name='opinel'): +def read_creds(profile_name, csv_file=None, mfa_serial_arg=None, mfa_code=None, force_init=False): """ Read credentials from anywhere (CSV, Environment, Instance metadata, config/credentials) @@ -442,7 +435,6 @@ def read_creds(profile_name, csv_file=None, mfa_serial_arg=None, mfa_code=None, :param mfa_serial_arg: :param mfa_code: :param force_init: - :param role_session_name: :return: """ @@ -504,7 +496,7 @@ def read_creds(profile_name, csv_file=None, mfa_serial_arg=None, mfa_code=None, credentials['TokenCode'] = prompt_mfa_code() if external_id: credentials['ExternalId'] = external_id - credentials = assume_role(profile_name, credentials, role_arn, role_session_name) + credentials = assume_role(profile_name, credentials, role_arn) # Read from ~/.aws/credentials else: credentials = read_creds_from_aws_credentials_file(profile_name) diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index 9394b85e9..74cf3cea5 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -20,7 +20,7 @@ class AWSProvider(BaseProvider): Implements provider for AWS """ - def __init__(self, profile=None, report_dir=None, timestamp=None, services=None, skipped_services=None, + def __init__(self, profile='default', report_dir=None, timestamp=None, services=None, skipped_services=None, thread_config=4, **kwargs): services = [] if services is None else services skipped_services = [] if skipped_services is None else skipped_services @@ -30,7 +30,7 @@ def __init__(self, profile=None, report_dir=None, timestamp=None, services=None, self.sg_map = {} self.subnet_map = {} - self.profile = profile[0] if profile else 'default' + self.profile = profile self.aws_account_id = None self.services_config = AWSServicesConfig @@ -44,7 +44,7 @@ def authenticate(self, profile, csv_credentials, mfa_serial, mfa_code, **kwargs) Implement authentication for the AWS provider :return: """ - self.credentials = read_creds(profile[0], csv_credentials, mfa_serial, mfa_code) + self.credentials = read_creds(profile, csv_credentials, mfa_serial, mfa_code) self.aws_account_id = get_aws_account_id(self.credentials) if self.credentials['AccessKeyId'] is None: From b703f4ea90e1686917f066a1b5c80b7fb1325181 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Thu, 28 Feb 2019 09:57:35 -0500 Subject: [PATCH 109/667] Removed return self --- ScoutSuite/providers/aws/services/ec2/vpcs.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ScoutSuite/providers/aws/services/ec2/vpcs.py b/ScoutSuite/providers/aws/services/ec2/vpcs.py index a73e28a6b..e80560b9c 100644 --- a/ScoutSuite/providers/aws/services/ec2/vpcs.py +++ b/ScoutSuite/providers/aws/services/ec2/vpcs.py @@ -23,8 +23,6 @@ async def fetch_all(self): scope = {'region': self.scope['region'], 'vpc': vpc} await self.fetch_children(self[vpc], scope=scope) - return self - def parse_resource(self, vpc): return vpc['VpcId'], {} From 85365d454b68574a2f2bf0808945713a83cd6e72 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Thu, 28 Feb 2019 10:03:48 -0500 Subject: [PATCH 110/667] Removed python 2.7 and 3.4 from travis --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index a5f2ca503..efa99608a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,8 +4,6 @@ language: python sudo: false python: - - "2.7" - - "3.4" - "3.5" - "3.6" From cb3bc31e28592bc9ba8ace658659e8f04d77dd3e Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Thu, 28 Feb 2019 11:36:07 -0500 Subject: [PATCH 111/667] Removed simpleresource --- ScoutSuite/providers/aws/resources/resources.py | 8 ++++---- .../providers/aws/services/awslambda/service.py | 4 ++-- ScoutSuite/providers/aws/services/ec2/ami.py | 4 ++-- ScoutSuite/providers/aws/services/ec2/instances.py | 4 ++-- .../providers/aws/services/ec2/networkinterfaces.py | 4 ++-- .../providers/aws/services/ec2/securitygroups.py | 4 ++-- ScoutSuite/providers/aws/services/ec2/snapshots.py | 4 ++-- ScoutSuite/providers/aws/services/ec2/volumes.py | 4 ++-- ScoutSuite/providers/aws/services/ec2/vpcs.py | 6 +++--- ScoutSuite/providers/base/configs/resources.py | 13 ------------- 10 files changed, 21 insertions(+), 34 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/resources.py b/ScoutSuite/providers/aws/resources/resources.py index 03fb85df3..a804cae66 100644 --- a/ScoutSuite/providers/aws/resources/resources.py +++ b/ScoutSuite/providers/aws/resources/resources.py @@ -1,15 +1,14 @@ -from ScoutSuite.providers.base.configs.resources import SimpleResources -from ScoutSuite.providers.base.configs.resources import CompositeResources +from ScoutSuite.providers.base.configs.resources import Resources, CompositeResources from ScoutSuite.providers.aws.facade.facade import AWSFacade from ScoutSuite.providers.aws.aws import get_aws_account_id import abc -class AWSSimpleResources(SimpleResources, metaclass=abc.ABCMeta): +class AWSResources(Resources, metaclass=abc.ABCMeta): def __init__(self, scope): self.scope = scope self.facade = AWSFacade() - async def fetch_all(self): + async def fetch_all(self, **kwargs): raw_resources = await self.get_resources_from_api() for raw_resource in raw_resources: name, resource = self.parse_resource(raw_resource) @@ -32,6 +31,7 @@ def __init__(self, service): self.facade = AWSFacade() async def fetch_all(self, credentials, chosen_regions=None, partition_name='aws'): + self['regions'] = {} for region in await self.facade.build_region_list(self.service, chosen_regions, partition_name): self['regions'][region] = { diff --git a/ScoutSuite/providers/aws/services/awslambda/service.py b/ScoutSuite/providers/aws/services/awslambda/service.py index d7d69e7c8..372a4ae93 100644 --- a/ScoutSuite/providers/aws/services/awslambda/service.py +++ b/ScoutSuite/providers/aws/services/awslambda/service.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- -from ScoutSuite.providers.aws.resources.resources import Regions, AWSSimpleResources +from ScoutSuite.providers.aws.resources.resources import Regions, AWSResources from ScoutSuite.providers.aws.facade.facade import AWSFacade from ScoutSuite.providers.aws.aws import build_region_list -class RegionalLambdas(AWSSimpleResources): +class RegionalLambdas(AWSResources): async def get_resources_from_api(self): return self.facade.awslambda.get_functions(self.scope['region']) diff --git a/ScoutSuite/providers/aws/services/ec2/ami.py b/ScoutSuite/providers/aws/services/ec2/ami.py index 1fea6ef44..dbafa5409 100644 --- a/ScoutSuite/providers/aws/services/ec2/ami.py +++ b/ScoutSuite/providers/aws/services/ec2/ami.py @@ -1,8 +1,8 @@ -from ScoutSuite.providers.aws.resources.resources import AWSSimpleResources +from ScoutSuite.providers.aws.resources.resources import AWSResources from ScoutSuite.providers.aws.facade.facade import AWSFacade -class AmazonMachineImages(AWSSimpleResources): +class AmazonMachineImages(AWSResources): async def get_resources_from_api(self): return self.facade.ec2.get_images(self.scope['region'], self.scope['owner_id']) diff --git a/ScoutSuite/providers/aws/services/ec2/instances.py b/ScoutSuite/providers/aws/services/ec2/instances.py index 3217abacd..106926547 100644 --- a/ScoutSuite/providers/aws/services/ec2/instances.py +++ b/ScoutSuite/providers/aws/services/ec2/instances.py @@ -1,10 +1,10 @@ -from ScoutSuite.providers.aws.resources.resources import AWSSimpleResources +from ScoutSuite.providers.aws.resources.resources import AWSResources from ScoutSuite.providers.aws.facade.facade import AWSFacade from ScoutSuite.providers.aws.aws import get_name from ScoutSuite.providers.aws.utils import ec2_classic, get_keys -class EC2Instances(AWSSimpleResources): +class EC2Instances(AWSResources): async def get_resources_from_api(self): return self.facade.ec2.get_instances(self.scope['region'], self.scope['vpc']) diff --git a/ScoutSuite/providers/aws/services/ec2/networkinterfaces.py b/ScoutSuite/providers/aws/services/ec2/networkinterfaces.py index 0e8abb320..dbdca6770 100644 --- a/ScoutSuite/providers/aws/services/ec2/networkinterfaces.py +++ b/ScoutSuite/providers/aws/services/ec2/networkinterfaces.py @@ -1,8 +1,8 @@ -from ScoutSuite.providers.aws.resources.resources import AWSSimpleResources +from ScoutSuite.providers.aws.resources.resources import AWSResources from ScoutSuite.providers.aws.facade.facade import AWSFacade -class NetworkInterfaces(AWSSimpleResources): +class NetworkInterfaces(AWSResources): async def get_resources_from_api(self): return self.facade.ec2.get_network_interfaces(self.scope['region'], self.scope['vpc']) diff --git a/ScoutSuite/providers/aws/services/ec2/securitygroups.py b/ScoutSuite/providers/aws/services/ec2/securitygroups.py index ee6edeabc..246f4dd83 100644 --- a/ScoutSuite/providers/aws/services/ec2/securitygroups.py +++ b/ScoutSuite/providers/aws/services/ec2/securitygroups.py @@ -1,4 +1,4 @@ -from ScoutSuite.providers.aws.resources.resources import AWSSimpleResources +from ScoutSuite.providers.aws.resources.resources import AWSResources from ScoutSuite.providers.aws.facade.facade import AWSFacade from ScoutSuite.providers.aws.aws import get_name from ScoutSuite.providers.aws.utils import ec2_classic, get_keys @@ -6,7 +6,7 @@ from ScoutSuite.core.fs import load_data -class SecurityGroups(AWSSimpleResources): +class SecurityGroups(AWSResources): icmp_message_types_dict = load_data( 'icmp_message_types.json', 'icmp_message_types') diff --git a/ScoutSuite/providers/aws/services/ec2/snapshots.py b/ScoutSuite/providers/aws/services/ec2/snapshots.py index b589a91f1..f12b9d6e3 100644 --- a/ScoutSuite/providers/aws/services/ec2/snapshots.py +++ b/ScoutSuite/providers/aws/services/ec2/snapshots.py @@ -1,9 +1,9 @@ -from ScoutSuite.providers.aws.resources.resources import AWSSimpleResources +from ScoutSuite.providers.aws.resources.resources import AWSResources from ScoutSuite.providers.aws.facade.facade import AWSFacade from ScoutSuite.providers.aws.aws import get_name -class Snapshots(AWSSimpleResources): +class Snapshots(AWSResources): async def get_resources_from_api(self): return self.facade.ec2.get_snapshots(self.scope['region'], self.scope['owner_id']) diff --git a/ScoutSuite/providers/aws/services/ec2/volumes.py b/ScoutSuite/providers/aws/services/ec2/volumes.py index 4b3017239..f67eec470 100644 --- a/ScoutSuite/providers/aws/services/ec2/volumes.py +++ b/ScoutSuite/providers/aws/services/ec2/volumes.py @@ -1,9 +1,9 @@ -from ScoutSuite.providers.aws.resources.resources import AWSSimpleResources +from ScoutSuite.providers.aws.resources.resources import AWSResources from ScoutSuite.providers.aws.facade.facade import AWSFacade from ScoutSuite.providers.aws.aws import get_name -class Volumes(AWSSimpleResources): +class Volumes(AWSResources): async def get_resources_from_api(self): return self.facade.ec2.get_volumes(self.scope['region']) diff --git a/ScoutSuite/providers/aws/services/ec2/vpcs.py b/ScoutSuite/providers/aws/services/ec2/vpcs.py index e80560b9c..68aca40e7 100644 --- a/ScoutSuite/providers/aws/services/ec2/vpcs.py +++ b/ScoutSuite/providers/aws/services/ec2/vpcs.py @@ -1,11 +1,11 @@ -from ScoutSuite.providers.aws.resources.resources import AWSCompositeResources, AWSSimpleResources +from ScoutSuite.providers.aws.resources.resources import AWSCompositeResources from ScoutSuite.providers.aws.facade.facade import AWSFacade from ScoutSuite.providers.aws.services.ec2.instances import EC2Instances from ScoutSuite.providers.aws.services.ec2.securitygroups import SecurityGroups from ScoutSuite.providers.aws.services.ec2.networkinterfaces import NetworkInterfaces -class Vpcs(AWSCompositeResources, AWSSimpleResources): +class Vpcs(AWSCompositeResources): children = [ (EC2Instances, 'instances'), (SecurityGroups, 'security_groups'), @@ -16,7 +16,7 @@ def __init__(self, scope): self.scope = scope self.facade = AWSFacade() - async def fetch_all(self): + async def fetch_all(self, **kwargs): await super(Vpcs, self).fetch_all() for vpc in self: diff --git a/ScoutSuite/providers/base/configs/resources.py b/ScoutSuite/providers/base/configs/resources.py index c75c01766..0d6b28f4a 100644 --- a/ScoutSuite/providers/base/configs/resources.py +++ b/ScoutSuite/providers/base/configs/resources.py @@ -4,25 +4,12 @@ class Resources(dict, metaclass=abc.ABCMeta): - @abc.abstractmethod async def fetch_all(self, **kwargs): raise NotImplementedError() -class SimpleResources(Resources, metaclass=abc.ABCMeta): - - @abc.abstractmethod - def parse_resource(self, resource): - raise NotImplementedError - - @abc.abstractmethod - async def get_resources_from_api(self, **kwargs): - raise NotImplementedError - - class CompositeResources(Resources, metaclass=abc.ABCMeta): - # The following enforces that classes which inherits from CompositeResources define a `children` attribute: @property @abc.abstractmethod From 0b63ddcb2505e7292fa6027c24a2e69db21f5236 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Thu, 28 Feb 2019 11:48:15 -0500 Subject: [PATCH 112/667] Added docstrings --- .../providers/base/configs/resources.py | 37 ++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/providers/base/configs/resources.py b/ScoutSuite/providers/base/configs/resources.py index 0d6b28f4a..bf2a23a75 100644 --- a/ScoutSuite/providers/base/configs/resources.py +++ b/ScoutSuite/providers/base/configs/resources.py @@ -1,21 +1,56 @@ # -*- coding: utf-8 -*- +""" +This module provides some abstract classes for representing a hierarchical structure. +Especially since all cloud providers (AWS, Azure and GCP for now) organize their resources (virtual machines, +databases, load balancers, user accounts and so on...) with some kind of hierarchy, these classes may be +used to reflect that. +""" import abc class Resources(dict, metaclass=abc.ABCMeta): + + """ + This is the base class of a hierarchical structure. Everything is basically `Resources`. + It stores in its internal dictionary instances of a given type of resources, with instance ids as keys and + instance configurations (which store other nested resources) as values. + """ + @abc.abstractmethod async def fetch_all(self, **kwargs): + """Fetches, parses and stores instances of a given type of resources from a cloud provider API. + + :param kwargs: + :return: + """ raise NotImplementedError() class CompositeResources(Resources, metaclass=abc.ABCMeta): - # The following enforces that classes which inherits from CompositeResources define a `children` attribute: + + """ + This class represents a node in the hierarchical structure. + As inherited from `Resources`, it still stores instances of a given type of resources internally but + also store some kind of nested resources referred as its 'children'. + """ + @property @abc.abstractmethod def children(self): + """A class that inherits from 'CompositeResources' should define a 'children' attribute, typically a list of + `Resources` classes. That is enforced by this abstract property. + + :return: + """ raise NotImplementedError @abc.abstractmethod async def fetch_children(self, **kwargs): + """Fetches, parses and stores instances of nested resources included in a `CompositeResources` and defined + in the 'children' attribute. + + :param kwargs: + :return: + """ raise NotImplementedError From 5dcf8dfd053ef2d588f1abffd0a1622f6f409ee3 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Thu, 28 Feb 2019 11:59:39 -0500 Subject: [PATCH 113/667] Updated resources.py --- .../providers/base/configs/resources.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/ScoutSuite/providers/base/configs/resources.py b/ScoutSuite/providers/base/configs/resources.py index d709c4c3c..bf2a23a75 100644 --- a/ScoutSuite/providers/base/configs/resources.py +++ b/ScoutSuite/providers/base/configs/resources.py @@ -1,10 +1,56 @@ # -*- coding: utf-8 -*- +""" +This module provides some abstract classes for representing a hierarchical structure. +Especially since all cloud providers (AWS, Azure and GCP for now) organize their resources (virtual machines, +databases, load balancers, user accounts and so on...) with some kind of hierarchy, these classes may be +used to reflect that. +""" import abc class Resources(dict, metaclass=abc.ABCMeta): + """ + This is the base class of a hierarchical structure. Everything is basically `Resources`. + It stores in its internal dictionary instances of a given type of resources, with instance ids as keys and + instance configurations (which store other nested resources) as values. + """ + @abc.abstractmethod async def fetch_all(self, **kwargs): + """Fetches, parses and stores instances of a given type of resources from a cloud provider API. + + :param kwargs: + :return: + """ raise NotImplementedError() + + +class CompositeResources(Resources, metaclass=abc.ABCMeta): + + """ + This class represents a node in the hierarchical structure. + As inherited from `Resources`, it still stores instances of a given type of resources internally but + also store some kind of nested resources referred as its 'children'. + """ + + @property + @abc.abstractmethod + def children(self): + """A class that inherits from 'CompositeResources' should define a 'children' attribute, typically a list of + `Resources` classes. That is enforced by this abstract property. + + :return: + """ + raise NotImplementedError + + @abc.abstractmethod + async def fetch_children(self, **kwargs): + """Fetches, parses and stores instances of nested resources included in a `CompositeResources` and defined + in the 'children' attribute. + + :param kwargs: + :return: + """ + raise NotImplementedError From 1bffe30fe53f3138b50e3956815f620b4ed31db1 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Thu, 28 Feb 2019 18:22:44 -0500 Subject: [PATCH 114/667] Reverted over-generalization --- ScoutSuite/providers/aws/resources/regions.py | 29 ++++++++++++++ .../providers/aws/resources/resources.py | 38 ++----------------- .../aws/services/awslambda/service.py | 12 ++++-- ScoutSuite/providers/aws/services/ec2/ami.py | 9 +++-- .../providers/aws/services/ec2/instances.py | 11 ++++-- .../aws/services/ec2/networkinterfaces.py | 9 +++-- .../aws/services/ec2/securitygroups.py | 12 +++--- .../providers/aws/services/ec2/service.py | 2 +- .../providers/aws/services/ec2/snapshots.py | 8 +++- .../providers/aws/services/ec2/volumes.py | 9 +++-- ScoutSuite/providers/aws/services/ec2/vpcs.py | 9 +++-- 11 files changed, 86 insertions(+), 62 deletions(-) create mode 100644 ScoutSuite/providers/aws/resources/regions.py diff --git a/ScoutSuite/providers/aws/resources/regions.py b/ScoutSuite/providers/aws/resources/regions.py new file mode 100644 index 000000000..ef24eebb7 --- /dev/null +++ b/ScoutSuite/providers/aws/resources/regions.py @@ -0,0 +1,29 @@ +from ScoutSuite.providers.aws.aws import get_aws_account_id +from ScoutSuite.providers.aws.resources.resources import AWSCompositeResources +from ScoutSuite.providers.aws.facade.facade import AWSFacade +import abc + +class Regions(AWSCompositeResources, metaclass=abc.ABCMeta): + def __init__(self, service): + self.service = service + # TODO: Should be injected + self.facade = AWSFacade() + + async def fetch_all(self, credentials, chosen_regions=None, partition_name='aws'): + + self['regions'] = {} + for region in await self.facade.build_region_list(self.service, chosen_regions, partition_name): + self['regions'][region] = { + 'id': region, + 'region': region, + 'name': region + } + + await self.fetch_children(self['regions'][region], scope={'region': region, 'owner_id': get_aws_account_id(credentials)}) + + self._set_counts() + + def _set_counts(self): + self['regions_count'] = len(self['regions']) + for _, key in self.children: + self[key + '_count'] = sum([region[key + '_count'] for region in self['regions'].values()]) diff --git a/ScoutSuite/providers/aws/resources/resources.py b/ScoutSuite/providers/aws/resources/resources.py index a804cae66..33eb971d7 100644 --- a/ScoutSuite/providers/aws/resources/resources.py +++ b/ScoutSuite/providers/aws/resources/resources.py @@ -1,50 +1,20 @@ from ScoutSuite.providers.base.configs.resources import Resources, CompositeResources from ScoutSuite.providers.aws.facade.facade import AWSFacade -from ScoutSuite.providers.aws.aws import get_aws_account_id import abc + class AWSResources(Resources, metaclass=abc.ABCMeta): def __init__(self, scope): self.scope = scope self.facade = AWSFacade() - async def fetch_all(self, **kwargs): - raw_resources = await self.get_resources_from_api() - for raw_resource in raw_resources: - name, resource = self.parse_resource(raw_resource) - self[name] = resource -class AWSCompositeResources(CompositeResources, metaclass=abc.ABCMeta): +class AWSCompositeResources(AWSResources, CompositeResources, metaclass=abc.ABCMeta): async def fetch_children(self, parent, **kwargs): for child_class, key in self.children: child = child_class(**kwargs) await child.fetch_all() - if parent.get(key) is None: + if parent.get(key) is None: parent[key] = {} parent[key].update(child) - parent[key + '_count'] = len(child) - -class Regions(AWSCompositeResources, metaclass=abc.ABCMeta): - def __init__(self, service): - self.service = service - # TODO: Should be injected - self.facade = AWSFacade() - - async def fetch_all(self, credentials, chosen_regions=None, partition_name='aws'): - - self['regions'] = {} - for region in await self.facade.build_region_list(self.service, chosen_regions, partition_name): - self['regions'][region] = { - 'id': region, - 'region': region, - 'name': region - } - - await self.fetch_children(self['regions'][region], scope={'region': region, 'owner_id': get_aws_account_id(credentials)}) - - self._set_counts() - - def _set_counts(self): - self['regions_count'] = len(self['regions']) - for _, key in self.children: - self[key + '_count'] = sum([region[key + '_count'] for region in self['regions'].values()]) + parent[key + '_count'] = len(child) \ No newline at end of file diff --git a/ScoutSuite/providers/aws/services/awslambda/service.py b/ScoutSuite/providers/aws/services/awslambda/service.py index 372a4ae93..f10d8dfc4 100644 --- a/ScoutSuite/providers/aws/services/awslambda/service.py +++ b/ScoutSuite/providers/aws/services/awslambda/service.py @@ -1,14 +1,18 @@ # -*- coding: utf-8 -*- -from ScoutSuite.providers.aws.resources.resources import Regions, AWSResources +from ScoutSuite.providers.aws.resources.regions import Regions +from ScoutSuite.providers.aws.resources.resources import AWSResources from ScoutSuite.providers.aws.facade.facade import AWSFacade from ScoutSuite.providers.aws.aws import build_region_list class RegionalLambdas(AWSResources): - async def get_resources_from_api(self): - return self.facade.awslambda.get_functions(self.scope['region']) + async def fetch_all(self, **kwargs): + raw_functions = self.facade.awslambda.get_functions(self.scope['region']) + for raw_function in raw_functions: + name, resource = self._parse_function(raw_function) + self[name] = resource - def parse_resource(self, raw_function): + def _parse_function(self, raw_function): raw_function['name'] = raw_function.pop('FunctionName') return (raw_function['name'], raw_function) diff --git a/ScoutSuite/providers/aws/services/ec2/ami.py b/ScoutSuite/providers/aws/services/ec2/ami.py index dbafa5409..7ba4c7f6a 100644 --- a/ScoutSuite/providers/aws/services/ec2/ami.py +++ b/ScoutSuite/providers/aws/services/ec2/ami.py @@ -3,10 +3,13 @@ class AmazonMachineImages(AWSResources): - async def get_resources_from_api(self): - return self.facade.ec2.get_images(self.scope['region'], self.scope['owner_id']) + async def fetch_all(self, **kwargs): + raw_images = self.facade.ec2.get_images(self.scope['region'], self.scope['owner_id']) + for raw_image in raw_images: + name, resource = self._parse_image(raw_image) + self[name] = resource - def parse_resource(self, raw_image): + def _parse_image(self, raw_image): raw_image['id'] = raw_image['ImageId'] raw_image['name'] = raw_image['Name'] diff --git a/ScoutSuite/providers/aws/services/ec2/instances.py b/ScoutSuite/providers/aws/services/ec2/instances.py index 106926547..9eb3a3a6e 100644 --- a/ScoutSuite/providers/aws/services/ec2/instances.py +++ b/ScoutSuite/providers/aws/services/ec2/instances.py @@ -5,10 +5,13 @@ class EC2Instances(AWSResources): - async def get_resources_from_api(self): - return self.facade.ec2.get_instances(self.scope['region'], self.scope['vpc']) - - def parse_resource(self, raw_instance): + async def fetch_all(self, **kwargs): + raw_instances = self.facade.ec2.get_instances(self.scope['region'], self.scope['vpc']) + for raw_instance in raw_instances: + name, resource = self._parse_instance(raw_instance) + self[name] = resource + + def _parse_instance(self, raw_instance): instance = {} id = raw_instance['InstanceId'] instance['id'] = id diff --git a/ScoutSuite/providers/aws/services/ec2/networkinterfaces.py b/ScoutSuite/providers/aws/services/ec2/networkinterfaces.py index dbdca6770..63408ddab 100644 --- a/ScoutSuite/providers/aws/services/ec2/networkinterfaces.py +++ b/ScoutSuite/providers/aws/services/ec2/networkinterfaces.py @@ -3,9 +3,12 @@ class NetworkInterfaces(AWSResources): - async def get_resources_from_api(self): - return self.facade.ec2.get_network_interfaces(self.scope['region'], self.scope['vpc']) + async def fetch_all(self, **kwargs): + raw_security_groups = self.facade.ec2.get_network_interfaces(self.scope['region'], self.scope['vpc']) + for raw_security_groups in raw_security_groups: + name, resource = self._parse_network_interface(raw_security_groups) + self[name] = resource - def parse_resource(self, raw_network_interace): + def _parse_network_interface(self, raw_network_interace): raw_network_interace['name'] = raw_network_interace['NetworkInterfaceId'] return raw_network_interace['NetworkInterfaceId'], raw_network_interace diff --git a/ScoutSuite/providers/aws/services/ec2/securitygroups.py b/ScoutSuite/providers/aws/services/ec2/securitygroups.py index 246f4dd83..c59b87129 100644 --- a/ScoutSuite/providers/aws/services/ec2/securitygroups.py +++ b/ScoutSuite/providers/aws/services/ec2/securitygroups.py @@ -7,13 +7,15 @@ class SecurityGroups(AWSResources): - icmp_message_types_dict = load_data( - 'icmp_message_types.json', 'icmp_message_types') + icmp_message_types_dict = load_data('icmp_message_types.json', 'icmp_message_types') - async def get_resources_from_api(self): - return self.facade.ec2.get_security_groups(self.scope['region'], self.scope['vpc']) + async def fetch_all(self, **kwargs): + raw_security_groups = self.facade.ec2.get_security_groups(self.scope['region'], self.scope['vpc']) + for raw_security_groups in raw_security_groups: + name, resource = self._parse_security_group(raw_security_groups) + self[name] = resource - def parse_resource(self, raw_security_group): + def _parse_security_group(self, raw_security_group): security_group = {} security_group['name'] = raw_security_group['GroupName'] security_group['id'] = raw_security_group['GroupId'] diff --git a/ScoutSuite/providers/aws/services/ec2/service.py b/ScoutSuite/providers/aws/services/ec2/service.py index fa38f80f9..37ac84af7 100644 --- a/ScoutSuite/providers/aws/services/ec2/service.py +++ b/ScoutSuite/providers/aws/services/ec2/service.py @@ -1,4 +1,4 @@ -from ScoutSuite.providers.aws.resources.resources import Regions +from ScoutSuite.providers.aws.resources.regions import Regions from ScoutSuite.providers.aws.services.ec2.ami import AmazonMachineImages from ScoutSuite.providers.aws.services.ec2.vpcs import Vpcs from ScoutSuite.providers.aws.services.ec2.snapshots import Snapshots diff --git a/ScoutSuite/providers/aws/services/ec2/snapshots.py b/ScoutSuite/providers/aws/services/ec2/snapshots.py index f12b9d6e3..1606bb8b9 100644 --- a/ScoutSuite/providers/aws/services/ec2/snapshots.py +++ b/ScoutSuite/providers/aws/services/ec2/snapshots.py @@ -4,10 +4,16 @@ class Snapshots(AWSResources): + async def fetch_all(self, **kwargs): + raw_snapshots = self.facade.ec2.get_snapshots(self.scope['region'], self.scope['owner_id']) + for raw_snapshot in raw_snapshots: + name, resource = self._parse_snapshot(raw_snapshot) + self[name] = resource + async def get_resources_from_api(self): return self.facade.ec2.get_snapshots(self.scope['region'], self.scope['owner_id']) - def parse_resource(self, raw_snapshot): + def _parse_snapshot(self, raw_snapshot): raw_snapshot['id'] = raw_snapshot.pop('SnapshotId') raw_snapshot['name'] = get_name(raw_snapshot, raw_snapshot, 'id') raw_snapshot['public'] = self._is_public(raw_snapshot) diff --git a/ScoutSuite/providers/aws/services/ec2/volumes.py b/ScoutSuite/providers/aws/services/ec2/volumes.py index f67eec470..16970fcf5 100644 --- a/ScoutSuite/providers/aws/services/ec2/volumes.py +++ b/ScoutSuite/providers/aws/services/ec2/volumes.py @@ -4,10 +4,13 @@ class Volumes(AWSResources): - async def get_resources_from_api(self): - return self.facade.ec2.get_volumes(self.scope['region']) + async def fetch_all(self, **kwargs): + raw_volumes = self.facade.ec2.get_volumes(self.scope['region']) + for raw_volume in raw_volumes: + name, resource = self._parse_volumes(raw_volume) + self[name] = resource - def parse_resource(self, raw_volume): + def _parse_volumes(self, raw_volume): raw_volume['id'] = raw_volume.pop('VolumeId') raw_volume['name'] = get_name(raw_volume, raw_volume, 'id') return raw_volume['id'], raw_volume diff --git a/ScoutSuite/providers/aws/services/ec2/vpcs.py b/ScoutSuite/providers/aws/services/ec2/vpcs.py index 68aca40e7..797276395 100644 --- a/ScoutSuite/providers/aws/services/ec2/vpcs.py +++ b/ScoutSuite/providers/aws/services/ec2/vpcs.py @@ -17,14 +17,15 @@ def __init__(self, scope): self.facade = AWSFacade() async def fetch_all(self, **kwargs): - await super(Vpcs, self).fetch_all() + vpcs = self.facade.ec2.get_vpcs(self.scope['region']) + for vpc in vpcs: + name, resource = self._parse_vpc(vpc) + self[name] = resource for vpc in self: scope = {'region': self.scope['region'], 'vpc': vpc} await self.fetch_children(self[vpc], scope=scope) - def parse_resource(self, vpc): + def _parse_vpc(self, vpc): return vpc['VpcId'], {} - async def get_resources_from_api(self): - return self.facade.ec2.get_vpcs(self.scope['region']) From 3d4bbb6f39a44f6618fb46477ff25e64bcabdf29 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Fri, 1 Mar 2019 10:26:35 -0500 Subject: [PATCH 115/667] Removed trailing lines --- ScoutSuite/providers/aws/facade/facade.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/facade.py b/ScoutSuite/providers/aws/facade/facade.py index 7f7f3b3b4..55ae28697 100644 --- a/ScoutSuite/providers/aws/facade/facade.py +++ b/ScoutSuite/providers/aws/facade/facade.py @@ -21,8 +21,4 @@ async def build_region_list(self, service, chosen_regions=None, partition_name=' if chosen_regions: return list((Counter(regions) & Counter(chosen_regions)).elements()) else: - return regions - - - - \ No newline at end of file + return regions \ No newline at end of file From 6ef8200f172204752be41384d24bcd8ebf67a6ac Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Fri, 1 Mar 2019 11:30:55 -0500 Subject: [PATCH 116/667] Fixed next marker problem --- ScoutSuite/providers/aws/facade/awslambda.py | 5 ++- ScoutSuite/providers/aws/facade/ec2.py | 46 ++++++++++---------- ScoutSuite/providers/aws/facade/utils.py | 10 +++-- 3 files changed, 34 insertions(+), 27 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/awslambda.py b/ScoutSuite/providers/aws/facade/awslambda.py index 711dc0fb8..8b9732508 100644 --- a/ScoutSuite/providers/aws/facade/awslambda.py +++ b/ScoutSuite/providers/aws/facade/awslambda.py @@ -9,6 +9,7 @@ class LambdaFacade: def get_functions(self, region): aws_lambda = boto3.client('lambda', region_name=region) return AWSFacadeUtils.get_all_pages( - lambda: aws_lambda.list_functions(), - lambda response: response['Functions'] + lambda marker: aws_lambda.list_functions(Marker=marker), + lambda response: response['Functions'], + 'Marker' ) diff --git a/ScoutSuite/providers/aws/facade/ec2.py b/ScoutSuite/providers/aws/facade/ec2.py index e27134cb6..5c4708d92 100644 --- a/ScoutSuite/providers/aws/facade/ec2.py +++ b/ScoutSuite/providers/aws/facade/ec2.py @@ -6,11 +6,10 @@ class EC2Facade: - def get_instance_user_data(self, region, instance_id): + def get_instance_user_data(self, region: str, instance_id: str): # TODO: We should save a list of the clients by region, as they are created. ec2_client = boto3.client('ec2', region_name=region) - user_data_response = ec2_client.describe_instance_attribute( - Attribute='userData', InstanceId=instance_id) + user_data_response = ec2_client.describe_instance_attribute(Attribute='userData', InstanceId=instance_id) if 'Value' not in user_data_response['UserData'].keys(): return None @@ -21,53 +20,56 @@ def get_instances(self, region, vpc): ec2_client = boto3.client('ec2', region_name=region) return AWSFacadeUtils.get_all_pages( - lambda: ec2_client.describe_instances(Filters=[{'Name': 'vpc-id', 'Values': [vpc]}]), - lambda response: itertools.chain.from_iterable( - [reservation['Instances'] for reservation in response['Reservations']]) + lambda next_token: ec2_client.describe_instances(Filters=[{'Name': 'vpc-id', 'Values': [vpc]}], NextToken=next_token), + lambda response: itertools.chain.from_iterable([reservation['Instances'] for reservation in response['Reservations']]), + 'NextToken' ) def get_security_groups(self, region, vpc): ec2_client = boto3.client('ec2', region_name=region) + filter = [{'Name': 'vpc-id', 'Values': [vpc]}] return AWSFacadeUtils.get_all_pages( - lambda: ec2_client.describe_security_groups(Filters=[{'Name': 'vpc-id', 'Values': [vpc]}]), - lambda response: response['SecurityGroups'] + lambda next_token: ec2_client.describe_security_groups(Filters=filter, NextToken=next_token), + lambda response: response['SecurityGroups'], + 'NextToken' ) def get_vpcs(self, region): ec2_client = boto3.client('ec2', region_name=region) - return AWSFacadeUtils.get_all_pages( - lambda: ec2_client.describe_vpcs(), - lambda response: response['Vpcs'] - ) + return ec2_client.describe_vpcs()['Vpcs'] def get_images(self, region, owner_id): ec2_client = boto3.client('ec2', region_name=region) - return AWSFacadeUtils.get_all_pages( - lambda: ec2_client .describe_images(Filters=[{'Name': 'owner-id', 'Values': [owner_id]}]), - lambda response: response['Images'] - ) + filter = [{'Name': 'owner-id', 'Values': [owner_id]}] + return ec2_client.describe_images(Filters=filter)['Images'] def get_network_interfaces(self, region, vpc): ec2_client = boto3.client('ec2', region_name=region) + filter = [{'Name': 'vpc-id', 'Values': [vpc]}] + return AWSFacadeUtils.get_all_pages( - lambda: ec2_client.describe_network_interfaces(Filters=[{'Name': 'vpc-id', 'Values': [vpc]}]), - lambda response: response['NetworkInterfaces'] + lambda next_token: ec2_client.describe_network_interfaces(Filters=filter, NextToken=next_token), + lambda response: response['NetworkInterfaces'], + 'NextToken' ) def get_volumes(self, region): ec2_client = boto3.client('ec2', region_name=region) return AWSFacadeUtils.get_all_pages( - lambda: ec2_client.describe_volumes(), - lambda response: response['Volumes'] + lambda next_token: ec2_client.describe_volumes(NextToken=next_token), + lambda response: response['Volumes'], + 'NextToken' ) def get_snapshots(self, region, owner_id): ec2_client = boto3.client('ec2', region_name=region) + filter = [{'Name': 'owner-id', 'Values': [owner_id]}] snapshots = AWSFacadeUtils.get_all_pages( - lambda: ec2_client.describe_snapshots(Filters=[{'Name': 'owner-id', 'Values': [owner_id]}]), - lambda response: response['Snapshots'] + lambda next_token: ec2_client.describe_snapshots(Filters=filter, NextToken=next_token), + lambda response: response['Snapshots'], + 'NextToken' ) for snapshot in snapshots: diff --git a/ScoutSuite/providers/aws/facade/utils.py b/ScoutSuite/providers/aws/facade/utils.py index f29d6f14c..999bc9350 100644 --- a/ScoutSuite/providers/aws/facade/utils.py +++ b/ScoutSuite/providers/aws/facade/utils.py @@ -1,14 +1,18 @@ +from typing import Callable + + class AWSFacadeUtils: @staticmethod - def get_all_pages(api_call, parse_response): + def get_all_pages(get_resources: Callable[[str], object], parse_response: Callable[[], object], next_page_marker_key: str): resources = [] + marker = '' while True: - response = api_call() + response = get_resources(marker) resources.extend(parse_response(response)) # TODO: this marker should be passed to the api call. Also, some calls return a NextMarker, some return a NextToken. - marker = response.get('NextMarker', None) + marker = response.get(next_page_marker_key, None) if marker is None: break From fea2cd418d460144af1dba7b6639054aafe4f773 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Fri, 1 Mar 2019 12:31:47 -0500 Subject: [PATCH 117/667] Changed pagination implementation to use paginator --- ScoutSuite/providers/aws/facade/awslambda.py | 7 +-- ScoutSuite/providers/aws/facade/ec2.py | 65 ++++++-------------- ScoutSuite/providers/aws/facade/facade.py | 2 +- ScoutSuite/providers/aws/facade/utils.py | 30 +++++---- 4 files changed, 39 insertions(+), 65 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/awslambda.py b/ScoutSuite/providers/aws/facade/awslambda.py index 8b9732508..adb8cfeca 100644 --- a/ScoutSuite/providers/aws/facade/awslambda.py +++ b/ScoutSuite/providers/aws/facade/awslambda.py @@ -7,9 +7,4 @@ class LambdaFacade: def get_functions(self, region): - aws_lambda = boto3.client('lambda', region_name=region) - return AWSFacadeUtils.get_all_pages( - lambda marker: aws_lambda.list_functions(Marker=marker), - lambda response: response['Functions'], - 'Marker' - ) + return AWSFacadeUtils.get_all_pages('lambda', region, 'list_functions', 'Functions') diff --git a/ScoutSuite/providers/aws/facade/ec2.py b/ScoutSuite/providers/aws/facade/ec2.py index 5c4708d92..975225878 100644 --- a/ScoutSuite/providers/aws/facade/ec2.py +++ b/ScoutSuite/providers/aws/facade/ec2.py @@ -7,8 +7,7 @@ class EC2Facade: def get_instance_user_data(self, region: str, instance_id: str): - # TODO: We should save a list of the clients by region, as they are created. - ec2_client = boto3.client('ec2', region_name=region) + ec2_client = AWSFacadeUtils.get_client('ec2', region) user_data_response = ec2_client.describe_instance_attribute(Attribute='userData', InstanceId=instance_id) if 'Value' not in user_data_response['UserData'].keys(): @@ -17,66 +16,40 @@ def get_instance_user_data(self, region: str, instance_id: str): return base64.b64decode(user_data_response['UserData']['Value']).decode('utf-8') def get_instances(self, region, vpc): - ec2_client = boto3.client('ec2', region_name=region) - - return AWSFacadeUtils.get_all_pages( - lambda next_token: ec2_client.describe_instances(Filters=[{'Name': 'vpc-id', 'Values': [vpc]}], NextToken=next_token), - lambda response: itertools.chain.from_iterable([reservation['Instances'] for reservation in response['Reservations']]), - 'NextToken' - ) + filters = [{'Name': 'vpc-id', 'Values': [vpc]}] + reservations = AWSFacadeUtils.get_all_pages('ec2', region, 'describe_instances', 'Reservations', Filters=filters) + return itertools.chain.from_iterable([reservation['Instances'] for reservation in reservations]) def get_security_groups(self, region, vpc): - ec2_client = boto3.client('ec2', region_name=region) - filter = [{'Name': 'vpc-id', 'Values': [vpc]}] - - return AWSFacadeUtils.get_all_pages( - lambda next_token: ec2_client.describe_security_groups(Filters=filter, NextToken=next_token), - lambda response: response['SecurityGroups'], - 'NextToken' - ) + filters = [{'Name': 'vpc-id', 'Values': [vpc]}] + return AWSFacadeUtils.get_all_pages('ec2', region, 'describe_security_groups', 'SecurityGroups', Filters=filters) def get_vpcs(self, region): - ec2_client = boto3.client('ec2', region_name=region) + ec2_client = boto3.client('ec2', region_name=region) return ec2_client.describe_vpcs()['Vpcs'] def get_images(self, region, owner_id): - ec2_client = boto3.client('ec2', region_name=region) - filter = [{'Name': 'owner-id', 'Values': [owner_id]}] - return ec2_client.describe_images(Filters=filter)['Images'] + filters = [{'Name': 'owner-id', 'Values': [owner_id]}] + response = AWSFacadeUtils.get_client('ec2', region) \ + .describe_images(Filters=filters) - def get_network_interfaces(self, region, vpc): - ec2_client = boto3.client('ec2', region_name=region) - filter = [{'Name': 'vpc-id', 'Values': [vpc]}] + return response['Images'] - return AWSFacadeUtils.get_all_pages( - lambda next_token: ec2_client.describe_network_interfaces(Filters=filter, NextToken=next_token), - lambda response: response['NetworkInterfaces'], - 'NextToken' - ) + def get_network_interfaces(self, region, vpc): + filters = [{'Name': 'vpc-id', 'Values': [vpc]}] + return AWSFacadeUtils.get_all_pages('ec2', region, 'describe_network_interfaces', 'NetworkInterfaces', Filters=filters) def get_volumes(self, region): - ec2_client = boto3.client('ec2', region_name=region) - return AWSFacadeUtils.get_all_pages( - lambda next_token: ec2_client.describe_volumes(NextToken=next_token), - lambda response: response['Volumes'], - 'NextToken' - ) - + return AWSFacadeUtils.get_all_pages('ec2', region, 'describe_volumes', 'Volumes') def get_snapshots(self, region, owner_id): - ec2_client = boto3.client('ec2', region_name=region) - filter = [{'Name': 'owner-id', 'Values': [owner_id]}] - snapshots = AWSFacadeUtils.get_all_pages( - lambda next_token: ec2_client.describe_snapshots(Filters=filter, NextToken=next_token), - lambda response: response['Snapshots'], - 'NextToken' - ) + filters = [{'Name': 'owner-id', 'Values': [owner_id]}] + snapshots = AWSFacadeUtils.get_all_pages('ec2', region, 'describe_snapshots', 'Snapshots', Filters=filters) + ec2_client = AWSFacadeUtils.get_client('ec2', region) for snapshot in snapshots: snapshot['CreateVolumePermissions'] = ec2_client.describe_snapshot_attribute( - Attribute='createVolumePermission', + Attribute='createVolumePermission', SnapshotId=snapshot['SnapshotId'])['CreateVolumePermissions'] return snapshots - - \ No newline at end of file diff --git a/ScoutSuite/providers/aws/facade/facade.py b/ScoutSuite/providers/aws/facade/facade.py index 55ae28697..baf02896d 100644 --- a/ScoutSuite/providers/aws/facade/facade.py +++ b/ScoutSuite/providers/aws/facade/facade.py @@ -9,7 +9,7 @@ def __init__(self): self.awslambda = LambdaFacade() - async def build_region_list(self, service, chosen_regions=None, partition_name='aws'): + async def build_region_list(self, service: str, chosen_regions=None, partition_name='aws'): service = 'ec2containerservice' if service == 'ecs' else service available_services = Session().get_available_services() diff --git a/ScoutSuite/providers/aws/facade/utils.py b/ScoutSuite/providers/aws/facade/utils.py index 999bc9350..19db9aa9e 100644 --- a/ScoutSuite/providers/aws/facade/utils.py +++ b/ScoutSuite/providers/aws/facade/utils.py @@ -1,19 +1,25 @@ from typing import Callable - +import boto3 class AWSFacadeUtils: + _clients = {} + @staticmethod - def get_all_pages(get_resources: Callable[[str], object], parse_response: Callable[[], object], next_page_marker_key: str): - resources = [] + def get_all_pages(service, region, paginator_name: str, response_key: str, **paginator_args): + pages = AWSFacadeUtils.get_client(service, region) \ + .get_paginator(paginator_name) \ + .paginate(**paginator_args) + + return AWSFacadeUtils._get_from_all_pages(pages, response_key) - marker = '' - while True: - response = get_resources(marker) - resources.extend(parse_response(response)) - - # TODO: this marker should be passed to the api call. Also, some calls return a NextMarker, some return a NextToken. - marker = response.get(next_page_marker_key, None) - if marker is None: - break + @staticmethod + def _get_from_all_pages(pages: [], key:str): + resources = [] + for page in pages: + resources.extend(page[key]) return resources + + @staticmethod + def get_client(service: str, region: str): + return AWSFacadeUtils._clients.setdefault((service, region), boto3.client(service, region_name=region)) \ No newline at end of file From 9a07e11e446c73f5db104add07ef5d926fc4f4d3 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Fri, 1 Mar 2019 12:32:54 -0500 Subject: [PATCH 118/667] Added TODO --- ScoutSuite/providers/aws/facade/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ScoutSuite/providers/aws/facade/utils.py b/ScoutSuite/providers/aws/facade/utils.py index 19db9aa9e..3b9d6bc8f 100644 --- a/ScoutSuite/providers/aws/facade/utils.py +++ b/ScoutSuite/providers/aws/facade/utils.py @@ -1,6 +1,7 @@ from typing import Callable import boto3 +# TODO: Add docstrings class AWSFacadeUtils: _clients = {} From bb8b2c4fdfc6aa8dc5c30c87819b3d54a006467d Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Fri, 1 Mar 2019 12:37:11 -0500 Subject: [PATCH 119/667] Removed todos --- ScoutSuite/providers/aws/services/awslambda/service.py | 1 - ScoutSuite/providers/aws/services/ec2/instances.py | 1 - ScoutSuite/providers/aws/services/ec2/service.py | 1 - 3 files changed, 3 deletions(-) diff --git a/ScoutSuite/providers/aws/services/awslambda/service.py b/ScoutSuite/providers/aws/services/awslambda/service.py index f10d8dfc4..6abaa559d 100644 --- a/ScoutSuite/providers/aws/services/awslambda/service.py +++ b/ScoutSuite/providers/aws/services/awslambda/service.py @@ -25,6 +25,5 @@ class Lambdas(Regions): def __init__(self): super(Lambdas, self).__init__('lambda') - # TODO: Remove the credentials parameter. We had to keep it for compatibility async def fetch_all(self, credentials=None, regions=None, partition_name='aws'): await super(Lambdas, self).fetch_all(credentials, regions, partition_name) diff --git a/ScoutSuite/providers/aws/services/ec2/instances.py b/ScoutSuite/providers/aws/services/ec2/instances.py index 9eb3a3a6e..229a42725 100644 --- a/ScoutSuite/providers/aws/services/ec2/instances.py +++ b/ScoutSuite/providers/aws/services/ec2/instances.py @@ -18,7 +18,6 @@ def _parse_instance(self, raw_instance): instance['monitoring_enabled'] = raw_instance['Monitoring']['State'] == 'enabled' instance['user_data'] = self.facade.ec2.get_instance_user_data(self.scope['region'], id) - # TODO: Those methods are slightly sketchy in my opinion (get methods which set stuff in a dictionary, say what) get_name(raw_instance, instance, 'InstanceId') get_keys(raw_instance, instance, ['KeyName', 'LaunchTime', 'InstanceType', 'State', 'IamInstanceProfile', 'SubnetId']) diff --git a/ScoutSuite/providers/aws/services/ec2/service.py b/ScoutSuite/providers/aws/services/ec2/service.py index 37ac84af7..4e6885fd7 100644 --- a/ScoutSuite/providers/aws/services/ec2/service.py +++ b/ScoutSuite/providers/aws/services/ec2/service.py @@ -22,7 +22,6 @@ async def fetch_all(self, credentials=None, regions=None, partition_name='aws'): await super(EC2, self).fetch_all(credentials, regions, partition_name) for region in self['regions']: - # TODO: Is there a way to move this elsewhere? self['regions'][region]['instances_count'] = sum([len(vpc['instances']) for vpc in self['regions'][region]['vpcs'].values()]) self['regions'][region]['security_groups_count'] = sum([len(vpc['security_groups']) for vpc in self['regions'][region]['vpcs'].values()]) self['regions'][region]['network_interfaces_count'] = sum([len(vpc['network_interfaces']) for vpc in self['regions'][region]['vpcs'].values()]) From facf8d5cedc1175a06ce37669d1b789968a07f61 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Fri, 1 Mar 2019 15:25:28 -0500 Subject: [PATCH 120/667] Removed redundant init --- ScoutSuite/providers/aws/services/ec2/vpcs.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ScoutSuite/providers/aws/services/ec2/vpcs.py b/ScoutSuite/providers/aws/services/ec2/vpcs.py index 797276395..2e259298f 100644 --- a/ScoutSuite/providers/aws/services/ec2/vpcs.py +++ b/ScoutSuite/providers/aws/services/ec2/vpcs.py @@ -12,10 +12,6 @@ class Vpcs(AWSCompositeResources): (NetworkInterfaces, 'network_interfaces') ] - def __init__(self, scope): - self.scope = scope - self.facade = AWSFacade() - async def fetch_all(self, **kwargs): vpcs = self.facade.ec2.get_vpcs(self.scope['region']) for vpc in vpcs: From 8b0582387ec179e338a782350ddf20f97c0ee962 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Sat, 2 Mar 2019 13:38:43 -0500 Subject: [PATCH 121/667] Added AWSCompositeResource test and test data --- tests/data/aws-resources/dummy_resources.json | 74 +++++++++++++++++++ tests/test-aws_resources.py | 46 ++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 tests/data/aws-resources/dummy_resources.json create mode 100644 tests/test-aws_resources.py diff --git a/tests/data/aws-resources/dummy_resources.json b/tests/data/aws-resources/dummy_resources.json new file mode 100644 index 000000000..7228c481a --- /dev/null +++ b/tests/data/aws-resources/dummy_resources.json @@ -0,0 +1,74 @@ +{ + "0": { + "some_dummy_resources": { + "resource_a": { + "some_id": 1, + "from_scope": { + "region": "some_region", + "some_inner_scope": "0" + } + }, + "resource_b": { + "some_id": 2, + "from_scope": { + "region": "some_region", + "some_inner_scope": "0" + } + } + }, + "some_dummy_resources_count": 2, + "other_dummy_resources": { + "resource_a": { + "some_id": 1, + "from_scope": { + "region": "some_region", + "some_inner_scope": "0" + } + }, + "resource_b": { + "some_id": 2, + "from_scope": { + "region": "some_region", + "some_inner_scope": "0" + } + } + }, + "other_dummy_resources_count": 2 + }, + "1": { + "some_dummy_resources": { + "resource_a": { + "some_id": 1, + "from_scope": { + "region": "some_region", + "some_inner_scope": "1" + } + }, + "resource_b": { + "some_id": 2, + "from_scope": { + "region": "some_region", + "some_inner_scope": "1" + } + } + }, + "some_dummy_resources_count": 2, + "other_dummy_resources": { + "resource_a": { + "some_id": 1, + "from_scope": { + "region": "some_region", + "some_inner_scope": "1" + } + }, + "resource_b": { + "some_id": 2, + "from_scope": { + "region": "some_region", + "some_inner_scope": "1" + } + } + }, + "other_dummy_resources_count": 2 + } +} \ No newline at end of file diff --git a/tests/test-aws_resources.py b/tests/test-aws_resources.py new file mode 100644 index 000000000..04e2a360a --- /dev/null +++ b/tests/test-aws_resources.py @@ -0,0 +1,46 @@ +from unittest import TestCase +from ScoutSuite.providers.aws.resources.resources import AWSResources, AWSCompositeResources +import asyncio +import json +import os + +class DummyResources(AWSResources): + async def fetch_all(self): + self['resource_a'] = { 'some_id': 1, 'from_scope': self.scope } + self['resource_b'] = { 'some_id': 2, 'from_scope': self.scope } + +class DummyComposite(AWSCompositeResources): + children = [ + (DummyResources, 'some_dummy_resources'), + (DummyResources, 'other_dummy_resources') + ] + + async def fetch_all(self, **kwargs): + for key in range(2): + self[str(key)] = {} + + for key in self: + scope = { + 'region': self.scope['region'], + 'some_inner_scope': key + } + + await self._fetch_children(parent=self[key], scope=scope) + + +class TestAWSResources(TestCase): + test_dir = os.path.dirname(os.path.realpath(__file__)) + + def test_lel(self): + loop = loop = asyncio.new_event_loop() + composite = DummyComposite({'region': 'some_region'}) + loop.run_until_complete(composite.fetch_all()) + + with open(os.path.join(self.test_dir, 'data/aws-resources/dummy_resources.json')) as f: + expected_object = json.load(f) + + expected_json = json.dumps(expected_object) + actual_json = json.dumps(composite) + + assert (expected_json == actual_json) + \ No newline at end of file From be637fdb27c54a83bddebcc795fac9ffe985b561 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Sat, 2 Mar 2019 13:39:04 -0500 Subject: [PATCH 122/667] Prefixed fetch_children with underscore --- ScoutSuite/providers/aws/resources/regions.py | 2 +- ScoutSuite/providers/aws/resources/resources.py | 2 +- ScoutSuite/providers/aws/services/ec2/vpcs.py | 2 +- ScoutSuite/providers/base/configs/resources.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/regions.py b/ScoutSuite/providers/aws/resources/regions.py index ef24eebb7..3d78050c5 100644 --- a/ScoutSuite/providers/aws/resources/regions.py +++ b/ScoutSuite/providers/aws/resources/regions.py @@ -19,7 +19,7 @@ async def fetch_all(self, credentials, chosen_regions=None, partition_name='aws' 'name': region } - await self.fetch_children(self['regions'][region], scope={'region': region, 'owner_id': get_aws_account_id(credentials)}) + await self._fetch_children(self['regions'][region], scope={'region': region, 'owner_id': get_aws_account_id(credentials)}) self._set_counts() diff --git a/ScoutSuite/providers/aws/resources/resources.py b/ScoutSuite/providers/aws/resources/resources.py index 33eb971d7..88302b17e 100644 --- a/ScoutSuite/providers/aws/resources/resources.py +++ b/ScoutSuite/providers/aws/resources/resources.py @@ -10,7 +10,7 @@ def __init__(self, scope): class AWSCompositeResources(AWSResources, CompositeResources, metaclass=abc.ABCMeta): - async def fetch_children(self, parent, **kwargs): + async def _fetch_children(self, parent, **kwargs): for child_class, key in self.children: child = child_class(**kwargs) await child.fetch_all() diff --git a/ScoutSuite/providers/aws/services/ec2/vpcs.py b/ScoutSuite/providers/aws/services/ec2/vpcs.py index 2e259298f..9f7346222 100644 --- a/ScoutSuite/providers/aws/services/ec2/vpcs.py +++ b/ScoutSuite/providers/aws/services/ec2/vpcs.py @@ -20,7 +20,7 @@ async def fetch_all(self, **kwargs): for vpc in self: scope = {'region': self.scope['region'], 'vpc': vpc} - await self.fetch_children(self[vpc], scope=scope) + await self._fetch_children(self[vpc], scope=scope) def _parse_vpc(self, vpc): return vpc['VpcId'], {} diff --git a/ScoutSuite/providers/base/configs/resources.py b/ScoutSuite/providers/base/configs/resources.py index bf2a23a75..075f76caa 100644 --- a/ScoutSuite/providers/base/configs/resources.py +++ b/ScoutSuite/providers/base/configs/resources.py @@ -46,7 +46,7 @@ def children(self): raise NotImplementedError @abc.abstractmethod - async def fetch_children(self, **kwargs): + async def _fetch_children(self, **kwargs): """Fetches, parses and stores instances of nested resources included in a `CompositeResources` and defined in the 'children' attribute. From 1fce367ef01313073ec7cc75eddfe65cee3efe58 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Sat, 2 Mar 2019 13:40:35 -0500 Subject: [PATCH 123/667] Added aws resources test to travis config file --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index efa99608a..dd7cab3a0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,6 +34,7 @@ script: # - 'nosetests tests/test-utils.py' # - '[ "${TRAVIS_SECURE_ENV_VARS}" = "true" ] && nosetests tests/test-utils_sns.py || false' - nosetests --with-coverage tests/test-main.py + - nosetests --with-coverage tests/test-aws_resources.py - nosetests --with-coverage tests/test-rules-ruleset.py - nosetests --with-coverage tests/test-rules-processingengine.py - nosetests --with-coverage --nocapture tests/test-scoutsuite.py -a "!credential" From d18dfdfd11f04b7218714d26ee6ddc9fbb3816e0 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Sat, 2 Mar 2019 16:41:25 -0500 Subject: [PATCH 124/667] Added docstrings --- ScoutSuite/providers/aws/resources/regions.py | 2 +- .../providers/aws/resources/resources.py | 50 ++++++++++++++++--- .../providers/aws/services/ec2/service.py | 2 - 3 files changed, 43 insertions(+), 11 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/regions.py b/ScoutSuite/providers/aws/resources/regions.py index 3d78050c5..ccedec41c 100644 --- a/ScoutSuite/providers/aws/resources/regions.py +++ b/ScoutSuite/providers/aws/resources/regions.py @@ -19,7 +19,7 @@ async def fetch_all(self, credentials, chosen_regions=None, partition_name='aws' 'name': region } - await self._fetch_children(self['regions'][region], scope={'region': region, 'owner_id': get_aws_account_id(credentials)}) + await self._fetch_children(self['regions'][region], {'region': region, 'owner_id': get_aws_account_id(credentials)}) self._set_counts() diff --git a/ScoutSuite/providers/aws/resources/resources.py b/ScoutSuite/providers/aws/resources/resources.py index 88302b17e..1f987e82e 100644 --- a/ScoutSuite/providers/aws/resources/resources.py +++ b/ScoutSuite/providers/aws/resources/resources.py @@ -1,20 +1,54 @@ +""" +This module provides implementations for Resources and CompositeResources for AWS. +""" + from ScoutSuite.providers.base.configs.resources import Resources, CompositeResources from ScoutSuite.providers.aws.facade.facade import AWSFacade import abc class AWSResources(Resources, metaclass=abc.ABCMeta): - def __init__(self, scope): + + """ + This is the base class for AWS resources. + """ + + def __init__(self, scope: dict): + """ + :param scope: The scope holds the scope in which the resource is located. This usually means \ + at least a region, but can also contain a VPC id, an owner id, etc. It should be \ + used when fetching the data through the facade. + """ + self.scope = scope self.facade = AWSFacade() class AWSCompositeResources(AWSResources, CompositeResources, metaclass=abc.ABCMeta): - async def _fetch_children(self, parent, **kwargs): - for child_class, key in self.children: - child = child_class(**kwargs) + + """ + This class represents a collection of AWSResources. Classes extending AWSCompositeResources should \ + define a "children" attribute which consists of a list of tuples describing the children. The tuples \ + are expected to respect the following format: (, ). The child_name is used by \ + indicates the name under which the child will be stored in the parent object. + """ + + async def _fetch_children(self, parent: object, scope: dict): + """ + This method calls fetch_all on each child defined in "children" and stores the fetched resources \ + in the parent under the key associated with the child. It also creates a "_count" entry \ + for each child. + + :param parent: The object in which the children should be stored + :param scope: The scope passed to the children constructors + """ + + for child_class, child_name in self.children: + child = child_class(scope) await child.fetch_all() - if parent.get(key) is None: - parent[key] = {} - parent[key].update(child) - parent[key + '_count'] = len(child) \ No newline at end of file + + if parent.get(child_name) is None: + parent[child_name] = {} + + parent[child_name].update(child) + parent[child_name + '_count'] = len(child) diff --git a/ScoutSuite/providers/aws/services/ec2/service.py b/ScoutSuite/providers/aws/services/ec2/service.py index 4e6885fd7..f2ad5f6b5 100644 --- a/ScoutSuite/providers/aws/services/ec2/service.py +++ b/ScoutSuite/providers/aws/services/ec2/service.py @@ -5,8 +5,6 @@ from ScoutSuite.providers.aws.services.ec2.volumes import Volumes -# TODO: Add docstrings - class EC2(Regions): children = [ (Vpcs, 'vpcs'), From c5a7d702668aeb2c14699df232632bb00b3bc7b5 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Sat, 2 Mar 2019 16:53:47 -0500 Subject: [PATCH 125/667] Added reservation id to instances in facade get_instances --- ScoutSuite/providers/aws/facade/ec2.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/facade/ec2.py b/ScoutSuite/providers/aws/facade/ec2.py index 975225878..a884a1a70 100644 --- a/ScoutSuite/providers/aws/facade/ec2.py +++ b/ScoutSuite/providers/aws/facade/ec2.py @@ -18,7 +18,14 @@ def get_instance_user_data(self, region: str, instance_id: str): def get_instances(self, region, vpc): filters = [{'Name': 'vpc-id', 'Values': [vpc]}] reservations = AWSFacadeUtils.get_all_pages('ec2', region, 'describe_instances', 'Reservations', Filters=filters) - return itertools.chain.from_iterable([reservation['Instances'] for reservation in reservations]) + + instances = [] + for reservation in reservations: + for instance in reservation['Instances']: + instance['ReservationId'] = reservation['ReservationId'] + instances.append(instance) + + return instances def get_security_groups(self, region, vpc): filters = [{'Name': 'vpc-id', 'Values': [vpc]}] From c99b911419a371bf3becf036777c89f42283f918 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Sat, 2 Mar 2019 16:54:01 -0500 Subject: [PATCH 126/667] Added reservation id in instances model --- .../partials/aws/services.ec2.regions.id.vpcs.id.instances.html | 1 + ScoutSuite/providers/aws/services/ec2/instances.py | 1 + 2 files changed, 2 insertions(+) diff --git a/ScoutSuite/output/data/html/partials/aws/services.ec2.regions.id.vpcs.id.instances.html b/ScoutSuite/output/data/html/partials/aws/services.ec2.regions.id.vpcs.id.instances.html index adf65ea43..c176cb407 100644 --- a/ScoutSuite/output/data/html/partials/aws/services.ec2.regions.id.vpcs.id.instances.html +++ b/ScoutSuite/output/data/html/partials/aws/services.ec2.regions.id.vpcs.id.instances.html @@ -9,6 +9,7 @@

    Information

  • Region: {{region}}
  • VPC: {{get_value_at 'services.ec2.regions' region 'vpcs' vpc 'name'}} ({{vpc}})
  • ID: {{id}}
    • +
  • Reservation ID: {{reservation_id}}
  • Monitoring: {{convert_bool_to_enabled monitoring_enabled}}
  • Access Key name: {{KeyName}}
  • State: {{make_title State.Name}}
    • diff --git a/ScoutSuite/providers/aws/services/ec2/instances.py b/ScoutSuite/providers/aws/services/ec2/instances.py index 229a42725..c5b9bf709 100644 --- a/ScoutSuite/providers/aws/services/ec2/instances.py +++ b/ScoutSuite/providers/aws/services/ec2/instances.py @@ -15,6 +15,7 @@ def _parse_instance(self, raw_instance): instance = {} id = raw_instance['InstanceId'] instance['id'] = id + instance['reservation_id'] = raw_instance['ReservationId'] instance['monitoring_enabled'] = raw_instance['Monitoring']['State'] == 'enabled' instance['user_data'] = self.facade.ec2.get_instance_user_data(self.scope['region'], id) From d11914b7df55cf931320a603bbf3562d28a398e4 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Sat, 2 Mar 2019 23:45:50 -0500 Subject: [PATCH 127/667] Renamed children to _children --- ScoutSuite/providers/aws/resources/regions.py | 2 +- .../providers/aws/resources/resources.py | 19 +++++++------------ .../aws/services/awslambda/service.py | 2 +- .../providers/aws/services/ec2/service.py | 2 +- ScoutSuite/providers/aws/services/ec2/vpcs.py | 2 +- .../providers/base/configs/resources.py | 18 ++++++++---------- 6 files changed, 19 insertions(+), 26 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/regions.py b/ScoutSuite/providers/aws/resources/regions.py index ccedec41c..01c88302b 100644 --- a/ScoutSuite/providers/aws/resources/regions.py +++ b/ScoutSuite/providers/aws/resources/regions.py @@ -25,5 +25,5 @@ async def fetch_all(self, credentials, chosen_regions=None, partition_name='aws' def _set_counts(self): self['regions_count'] = len(self['regions']) - for _, key in self.children: + for _, key in self._children: self[key + '_count'] = sum([region[key + '_count'] for region in self['regions'].values()]) diff --git a/ScoutSuite/providers/aws/resources/resources.py b/ScoutSuite/providers/aws/resources/resources.py index 1f987e82e..1741885ff 100644 --- a/ScoutSuite/providers/aws/resources/resources.py +++ b/ScoutSuite/providers/aws/resources/resources.py @@ -1,6 +1,5 @@ -""" -This module provides implementations for Resources and CompositeResources for AWS. -""" + +"""This module provides implementations for Resources and CompositeResources for AWS.""" from ScoutSuite.providers.base.configs.resources import Resources, CompositeResources from ScoutSuite.providers.aws.facade.facade import AWSFacade @@ -9,9 +8,7 @@ class AWSResources(Resources, metaclass=abc.ABCMeta): - """ - This is the base class for AWS resources. - """ + """This is the base class for AWS resources.""" def __init__(self, scope: dict): """ @@ -26,16 +23,14 @@ def __init__(self, scope: dict): class AWSCompositeResources(AWSResources, CompositeResources, metaclass=abc.ABCMeta): - """ - This class represents a collection of AWSResources. Classes extending AWSCompositeResources should \ - define a "children" attribute which consists of a list of tuples describing the children. The tuples \ + """This class represents a collection of AWSResources. Classes extending AWSCompositeResources should \ + define a "_children" attribute which consists of a list of tuples describing the children. The tuples \ are expected to respect the following format: (, ). The child_name is used by \ indicates the name under which the child will be stored in the parent object. """ async def _fetch_children(self, parent: object, scope: dict): - """ - This method calls fetch_all on each child defined in "children" and stores the fetched resources \ + """This method calls fetch_all on each child defined in "_children" and stores the fetched resources \ in the parent under the key associated with the child. It also creates a "_count" entry \ for each child. @@ -43,7 +38,7 @@ async def _fetch_children(self, parent: object, scope: dict): :param scope: The scope passed to the children constructors """ - for child_class, child_name in self.children: + for child_class, child_name in self._children: child = child_class(scope) await child.fetch_all() diff --git a/ScoutSuite/providers/aws/services/awslambda/service.py b/ScoutSuite/providers/aws/services/awslambda/service.py index 6abaa559d..947f2b6cb 100644 --- a/ScoutSuite/providers/aws/services/awslambda/service.py +++ b/ScoutSuite/providers/aws/services/awslambda/service.py @@ -18,7 +18,7 @@ def _parse_function(self, raw_function): class Lambdas(Regions): - children = [ + _children = [ (RegionalLambdas, 'functions') ] diff --git a/ScoutSuite/providers/aws/services/ec2/service.py b/ScoutSuite/providers/aws/services/ec2/service.py index f2ad5f6b5..404b8906c 100644 --- a/ScoutSuite/providers/aws/services/ec2/service.py +++ b/ScoutSuite/providers/aws/services/ec2/service.py @@ -6,7 +6,7 @@ class EC2(Regions): - children = [ + _children = [ (Vpcs, 'vpcs'), (AmazonMachineImages, 'images'), (Snapshots, 'snapshots'), diff --git a/ScoutSuite/providers/aws/services/ec2/vpcs.py b/ScoutSuite/providers/aws/services/ec2/vpcs.py index 9f7346222..5ddba3fd4 100644 --- a/ScoutSuite/providers/aws/services/ec2/vpcs.py +++ b/ScoutSuite/providers/aws/services/ec2/vpcs.py @@ -6,7 +6,7 @@ class Vpcs(AWSCompositeResources): - children = [ + _children = [ (EC2Instances, 'instances'), (SecurityGroups, 'security_groups'), (NetworkInterfaces, 'network_interfaces') diff --git a/ScoutSuite/providers/base/configs/resources.py b/ScoutSuite/providers/base/configs/resources.py index 075f76caa..7ec5b2326 100644 --- a/ScoutSuite/providers/base/configs/resources.py +++ b/ScoutSuite/providers/base/configs/resources.py @@ -11,8 +11,7 @@ class Resources(dict, metaclass=abc.ABCMeta): - """ - This is the base class of a hierarchical structure. Everything is basically `Resources`. + """This is the base class of a hierarchical structure. Everything is basically `Resources`. It stores in its internal dictionary instances of a given type of resources, with instance ids as keys and instance configurations (which store other nested resources) as values. """ @@ -29,17 +28,16 @@ async def fetch_all(self, **kwargs): class CompositeResources(Resources, metaclass=abc.ABCMeta): - """ - This class represents a node in the hierarchical structure. - As inherited from `Resources`, it still stores instances of a given type of resources internally but - also store some kind of nested resources referred as its 'children'. + """This class represents a node in the hierarchical structure. As inherited from `Resources`, it still \ + stores instances of a given type of resources internally but also stores some kind of nested resources \ + referred to as its 'children'. """ @property @abc.abstractmethod - def children(self): - """A class that inherits from 'CompositeResources' should define a 'children' attribute, typically a list of - `Resources` classes. That is enforced by this abstract property. + def _children(self): + """A class that inherits from 'CompositeResources' should define a '_children' attribute, typically a list \ + of `Resources` classes, or of tuples containing at least a `Resources` class. That is enforced by this abstract property. :return: """ @@ -47,7 +45,7 @@ def children(self): @abc.abstractmethod async def _fetch_children(self, **kwargs): - """Fetches, parses and stores instances of nested resources included in a `CompositeResources` and defined + """Fetches, parses and stores instances of nested resources included in a `CompositeResources` and defined \ in the 'children' attribute. :param kwargs: From c72e0eb4aa64f9790de03012b22627548828db35 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Sun, 3 Mar 2019 00:46:42 -0500 Subject: [PATCH 128/667] Fixed build --- ScoutSuite/providers/aws/resources/regions.py | 4 ++-- ScoutSuite/providers/aws/services/awslambda/service.py | 5 ----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/regions.py b/ScoutSuite/providers/aws/resources/regions.py index 01c88302b..1f0b3a7c0 100644 --- a/ScoutSuite/providers/aws/resources/regions.py +++ b/ScoutSuite/providers/aws/resources/regions.py @@ -9,10 +9,10 @@ def __init__(self, service): # TODO: Should be injected self.facade = AWSFacade() - async def fetch_all(self, credentials, chosen_regions=None, partition_name='aws'): + async def fetch_all(self, credentials, regions=None, partition_name='aws'): self['regions'] = {} - for region in await self.facade.build_region_list(self.service, chosen_regions, partition_name): + for region in await self.facade.build_region_list(self.service, regions, partition_name): self['regions'][region] = { 'id': region, 'region': region, diff --git a/ScoutSuite/providers/aws/services/awslambda/service.py b/ScoutSuite/providers/aws/services/awslambda/service.py index 947f2b6cb..7ec5fbb35 100644 --- a/ScoutSuite/providers/aws/services/awslambda/service.py +++ b/ScoutSuite/providers/aws/services/awslambda/service.py @@ -1,8 +1,6 @@ -# -*- coding: utf-8 -*- from ScoutSuite.providers.aws.resources.regions import Regions from ScoutSuite.providers.aws.resources.resources import AWSResources from ScoutSuite.providers.aws.facade.facade import AWSFacade -from ScoutSuite.providers.aws.aws import build_region_list class RegionalLambdas(AWSResources): @@ -24,6 +22,3 @@ class Lambdas(Regions): def __init__(self): super(Lambdas, self).__init__('lambda') - - async def fetch_all(self, credentials=None, regions=None, partition_name='aws'): - await super(Lambdas, self).fetch_all(credentials, regions, partition_name) From 7cd811396274da2e76d64024d74334a54f565538 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 3 Mar 2019 12:17:55 -0500 Subject: [PATCH 129/667] Make children and fetch_children properties private. --- ScoutSuite/providers/base/configs/resources.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ScoutSuite/providers/base/configs/resources.py b/ScoutSuite/providers/base/configs/resources.py index bf2a23a75..98b432845 100644 --- a/ScoutSuite/providers/base/configs/resources.py +++ b/ScoutSuite/providers/base/configs/resources.py @@ -37,18 +37,18 @@ class CompositeResources(Resources, metaclass=abc.ABCMeta): @property @abc.abstractmethod - def children(self): - """A class that inherits from 'CompositeResources' should define a 'children' attribute, typically a list of - `Resources` classes. That is enforced by this abstract property. + def _children(self): + """A class that inherits from 'CompositeResources' should define a private '_children' attribute, typically a + list of `Resources` classes. That is enforced by this abstract property. :return: """ raise NotImplementedError @abc.abstractmethod - async def fetch_children(self, **kwargs): + async def _fetch_children(self, **kwargs): """Fetches, parses and stores instances of nested resources included in a `CompositeResources` and defined - in the 'children' attribute. + in the '_children' attribute. :param kwargs: :return: From 1c7544ae3825a218f0d18508708ab9393dfcdeb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sgha=C3=AFer?= Date: Sun, 3 Mar 2019 17:16:14 -0500 Subject: [PATCH 130/667] Added type hints Co-Authored-By: Aboisier --- ScoutSuite/providers/aws/facade/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/utils.py b/ScoutSuite/providers/aws/facade/utils.py index 3b9d6bc8f..85ffb2bc0 100644 --- a/ScoutSuite/providers/aws/facade/utils.py +++ b/ScoutSuite/providers/aws/facade/utils.py @@ -6,7 +6,7 @@ class AWSFacadeUtils: _clients = {} @staticmethod - def get_all_pages(service, region, paginator_name: str, response_key: str, **paginator_args): + def get_all_pages(service: str, region: str, paginator_name: str, response_key: str, **paginator_args): pages = AWSFacadeUtils.get_client(service, region) \ .get_paginator(paginator_name) \ .paginate(**paginator_args) @@ -23,4 +23,4 @@ def _get_from_all_pages(pages: [], key:str): @staticmethod def get_client(service: str, region: str): - return AWSFacadeUtils._clients.setdefault((service, region), boto3.client(service, region_name=region)) \ No newline at end of file + return AWSFacadeUtils._clients.setdefault((service, region), boto3.client(service, region_name=region)) From e64906286804a12440342ad9bcc19b10e70b076e Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Sun, 3 Mar 2019 17:35:46 -0500 Subject: [PATCH 131/667] Fixed tests --- tests/test-aws_resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test-aws_resources.py b/tests/test-aws_resources.py index 04e2a360a..eebea04ab 100644 --- a/tests/test-aws_resources.py +++ b/tests/test-aws_resources.py @@ -10,7 +10,7 @@ async def fetch_all(self): self['resource_b'] = { 'some_id': 2, 'from_scope': self.scope } class DummyComposite(AWSCompositeResources): - children = [ + _children = [ (DummyResources, 'some_dummy_resources'), (DummyResources, 'other_dummy_resources') ] From 2e96f1b7fc7a2c858a7f58c482b0a98b4de499e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Tue, 19 Feb 2019 16:52:01 -0500 Subject: [PATCH 132/667] Add support for (database) auditing policies resources. --- .../sql/database_blob_auditing_policies.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py diff --git a/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py b/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py new file mode 100644 index 000000000..1f1c1762a --- /dev/null +++ b/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +from azure.mgmt.sql import SqlManagementClient + +from ScoutSuite.providers.base.configs.resource_config import ResourceConfig + + +class DatabaseBlobAuditingPoliciesConfig(ResourceConfig): + + def __init__(self, resource_group_name, server_name, database_name): + self.resource_group_name = resource_group_name + self.server_name = server_name + self.database_name = database_name + + self.value = None + + async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): + # sdk container: + api = SqlManagementClient(credentials.credentials, credentials.subscription_id) + + self.value =\ + api.database_blob_auditing_policies.get(self.resource_group_name, self.server_name, self.database_name) + # TODO: await api.database_blob_auditing_policies.get(self.resource_group_name, self.server_name, self.database_name) From 04a36b207ba513bf4217601c6d6f4010d55da8ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Tue, 19 Feb 2019 16:52:26 -0500 Subject: [PATCH 133/667] Add support for (database) threat detection policy resources. --- .../sql/database_threat_detection_policies.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py diff --git a/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py b/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py new file mode 100644 index 000000000..536bbd420 --- /dev/null +++ b/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +from azure.mgmt.sql import SqlManagementClient + +from ScoutSuite.providers.base.configs.resource_config import ResourceConfig + + +class DatabaseThreatDetectionPoliciesConfig(ResourceConfig): + + def __init__(self, resource_group_name, server_name, database_name): + self.resource_group_name = resource_group_name + self.server_name = server_name + self.database_name = database_name + + self.value = None + + async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): + # sdk container: + api = SqlManagementClient(credentials.credentials, credentials.subscription_id) + + self.value =\ + api.database_threat_detection_policies.get(self.resource_group_name, self.server_name, self.database_name) + # TODO: await api.transparent_data_encryptions.get(self.resource_group_name, self.server_name, self.database_name) From 1cc9081656cb7847c17d978a8f0d9324e6ed01bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Tue, 19 Feb 2019 16:52:46 -0500 Subject: [PATCH 134/667] Add support for (database) replication link resources. --- .../azure/resources/sql/replication_links.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 ScoutSuite/providers/azure/resources/sql/replication_links.py diff --git a/ScoutSuite/providers/azure/resources/sql/replication_links.py b/ScoutSuite/providers/azure/resources/sql/replication_links.py new file mode 100644 index 000000000..1005f0f17 --- /dev/null +++ b/ScoutSuite/providers/azure/resources/sql/replication_links.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +from azure.mgmt.sql import SqlManagementClient + +from ScoutSuite.providers.base.configs.resource_config import ResourceConfig + + +class ReplicationLinksConfig(ResourceConfig): + + def __init__(self, resource_group_name, server_name, database_name): + self.resource_group_name = resource_group_name + self.server_name = server_name + self.database_name = database_name + + self.value = None + + async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): + # sdk container: + api = SqlManagementClient(credentials.credentials, credentials.subscription_id) + + self.value =\ + list(api.replication_links.list_by_database(self.resource_group_name, self.server_name, self.database_name)) + # TODO: await api.replication_links.list_by_databases(self.resource_group_name, self.server_name, self.database_name) From 7438f6b6f01fe4d4e4dcd744f510c77dae416127 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Tue, 19 Feb 2019 16:53:22 -0500 Subject: [PATCH 135/667] Add support for azure ad administrator resources. --- .../sql/server_azure_ad_administrators.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py diff --git a/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py b/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py new file mode 100644 index 000000000..f68f91e9d --- /dev/null +++ b/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- + +from azure.mgmt.sql import SqlManagementClient +from msrestazure.azure_exceptions import CloudError + +from ScoutSuite.providers.base.configs.resource_config import ResourceConfig + + +class ServerAzureAdAdministratorsConfig(ResourceConfig): + + def __init__(self, resource_group_name, server_name): + self.resource_group_name = resource_group_name + self.server_name = server_name + + self.value = None + + async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): + # sdk container: + api = SqlManagementClient(credentials.credentials, credentials.subscription_id) + + try: + self.value = \ + api.server_azure_ad_administrators.get(self.resource_group_name, self.server_name) + # TODO: await api.server_azure_ad_administrators.get(self.resource_group_name, self.server_name) + except CloudError: # no ad admin configured returns a 404 error + pass From 83925e71503ae153d59ea773e20600bfb15e1152 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Tue, 19 Feb 2019 16:53:46 -0500 Subject: [PATCH 136/667] Add support for (database) transparent data encryption resources. --- .../sql/transparent_data_encryptions.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py diff --git a/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py b/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py new file mode 100644 index 000000000..e2a582046 --- /dev/null +++ b/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- + +from azure.mgmt.sql import SqlManagementClient + +from ScoutSuite.providers.base.configs.resource_config import ResourceConfig + + +class TransparentDataEncryptionsConfig(ResourceConfig): + + def __init__(self, resource_group_name, server_name, database_name): + self.resource_group_name = resource_group_name + self.server_name = server_name + self.database_name = database_name + + self.value = None + + async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): + # sdk container: + api = SqlManagementClient(credentials.credentials, credentials.subscription_id) + + self.value =\ + api.transparent_data_encryptions.get(self.resource_group_name, self.server_name, self.database_name) + # TODO: await api.transparent_data_encryptions.get(self.resource_group_name, self.server_name, self.database_name) + From 3791e3be63d8aeaa26aa21dd17c367aa7231e249 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Tue, 19 Feb 2019 16:55:55 -0500 Subject: [PATCH 137/667] Add __init__ to python packages. --- ScoutSuite/providers/azure/resources/__init__.py | 0 ScoutSuite/providers/azure/resources/sql/__init__.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 ScoutSuite/providers/azure/resources/__init__.py create mode 100644 ScoutSuite/providers/azure/resources/sql/__init__.py diff --git a/ScoutSuite/providers/azure/resources/__init__.py b/ScoutSuite/providers/azure/resources/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ScoutSuite/providers/azure/resources/sql/__init__.py b/ScoutSuite/providers/azure/resources/sql/__init__.py new file mode 100644 index 000000000..e69de29bb From 2bfb67d9a302028d993a8e0866c5ac11678b0eec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Tue, 19 Feb 2019 16:56:45 -0500 Subject: [PATCH 138/667] Add get_non_provider_id utility. --- ScoutSuite/providers/azure/resources/utils.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 ScoutSuite/providers/azure/resources/utils.py diff --git a/ScoutSuite/providers/azure/resources/utils.py b/ScoutSuite/providers/azure/resources/utils.py new file mode 100644 index 000000000..3011743f7 --- /dev/null +++ b/ScoutSuite/providers/azure/resources/utils.py @@ -0,0 +1,14 @@ +from hashlib import sha1 + + +def get_non_provider_id(name): + """ + Not all resources have an ID and some services allow the use of "." in names, which break's Scout2's + recursion scheme if name is used as an ID. Use SHA1(name) instead. + + :param name: Name of the resource to + :return: SHA1(name) + """ + m = sha1() + m.update(name.encode('utf-8')) + return m.hexdigest() From d2a5542458419ba13a114ac5ba7e4db9e869ed08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Tue, 19 Feb 2019 16:57:33 -0500 Subject: [PATCH 139/667] Add support for (sql) database resources. --- .../azure/resources/sql/databases.py | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 ScoutSuite/providers/azure/resources/sql/databases.py diff --git a/ScoutSuite/providers/azure/resources/sql/databases.py b/ScoutSuite/providers/azure/resources/sql/databases.py new file mode 100644 index 000000000..21ac1eb0d --- /dev/null +++ b/ScoutSuite/providers/azure/resources/sql/databases.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- + +from azure.mgmt.sql import SqlManagementClient + +from ScoutSuite.providers.base.configs.resource_config import ResourceConfig + +from .database_blob_auditing_policies import DatabaseBlobAuditingPoliciesConfig +from .database_threat_detection_policies import DatabaseThreatDetectionPoliciesConfig +from .replication_links import ReplicationLinksConfig +from .transparent_data_encryptions import TransparentDataEncryptionsConfig + + +class DatabasesConfig(ResourceConfig): + # the following static methods will be used to parse the config of each resource nested in a database config + # (defined in 'children' attribute below): + @staticmethod + def _parse_auditing_config(config): + return { + 'auditing_enabled': config.value.state == "Enabled" + } + + @staticmethod + def _parse_threat_detection_config(config): + return { + 'threat_detection_enabled': config.value.state == "Enabled" + } + + @staticmethod + def _parse_replication_links_config(config): + return { + 'replication_configured': len(config.value) > 0 + } + + @staticmethod + def _parse_transparent_data_encryption_config(config): + return { + 'transparent_data_encryption_enabled': config.value.status == "Enabled" + } + + # register children resources: + children = { + 'Database Auditing Settings': (DatabaseBlobAuditingPoliciesConfig, _parse_auditing_config), + 'Database Threat Detection Policies': (DatabaseThreatDetectionPoliciesConfig, _parse_threat_detection_config), + 'Replication Links': (ReplicationLinksConfig, _parse_replication_links_config), + 'Transparent Data Encryptions': (TransparentDataEncryptionsConfig, _parse_transparent_data_encryption_config) + } + + def __init__(self, resource_group_name, server_name): + self.resource_group_name = resource_group_name + self.server_name = server_name + + self.databases = {} + + async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): + # sdk container: + api = SqlManagementClient(credentials.credentials, credentials.subscription_id) + + # TODO: for db in await api.databases.list_by_server(self.resource_group_name, self.server_name): + for db in api.databases.list_by_server(self.resource_group_name, self.server_name): + self.databases[db.name] = { + 'id': db.name, + } + + # put the following code in a fetch_children() parent method (typical method for a composite node)? + for (resource_config, resource_parser) in self.children.values(): + resource = resource_config(self.resource_group_name, self.server_name, db.name) + await resource.fetch_all(credentials) + for k, v in resource_parser.__func__(resource).items(): + self.databases[db.name][k] = v From ce801ffe4d98bbb51757f46bfaa97fddd647b276 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Tue, 19 Feb 2019 16:57:44 -0500 Subject: [PATCH 140/667] Add support for (sql) server resources. --- .../providers/azure/resources/sql/servers.py | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 ScoutSuite/providers/azure/resources/sql/servers.py diff --git a/ScoutSuite/providers/azure/resources/sql/servers.py b/ScoutSuite/providers/azure/resources/sql/servers.py new file mode 100644 index 000000000..f1cc2b572 --- /dev/null +++ b/ScoutSuite/providers/azure/resources/sql/servers.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- + +from azure.mgmt.sql import SqlManagementClient + +from ScoutSuite.providers.base.configs.resource_config import ResourceConfig +from ScoutSuite.providers.azure.utils import get_resource_group_name + +from .databases import DatabasesConfig +from .server_azure_ad_administrators import ServerAzureAdAdministratorsConfig +from ..utils import get_non_provider_id + + +class ServersConfig(ResourceConfig): + # the following static methods will be used to parse the config of each resource nested in a server config + # (defined in 'children' attribute below): + @staticmethod + def _parse_databases_config(config): + return { + 'databases': config.databases + } + + @staticmethod + def _parse_azure_ad_administrators_config(config): + return { + 'ad_admin_configured': config.value is not None + } + + # register children resources: + children = { + 'Databases': (DatabasesConfig, _parse_databases_config), + 'Azure AD Administrators': (ServerAzureAdAdministratorsConfig, _parse_azure_ad_administrators_config) + } + + def __init__(self): + self.servers = {} + + async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): + # sdk container: + api = SqlManagementClient(credentials.credentials, credentials.subscription_id) + + # for server in await api.servers.list(): + for server in api.servers.list(): + # parsing: + + id = get_non_provider_id(server.id) + resource_group_name = get_resource_group_name(server.id) + + self.servers[id] = { + 'id': id, + 'name': server.name + } + + # put the following code in a fetch_children() parent method (typical method for a composite node)? + for (resource_config, resource_parser) in self.children.values(): + resource = resource_config(resource_group_name, server.name) + await resource.fetch_all(credentials) + for k, v in resource_parser.__func__(resource).items(): + self.servers[id][k] = v From 8c676c6fa745087cdfe390f55a66ee53ee12ea30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Tue, 19 Feb 2019 16:58:46 -0500 Subject: [PATCH 141/667] Switch SQLDatabase service implementation to the new one. --- ScoutSuite/providers/azure/configs/services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/azure/configs/services.py b/ScoutSuite/providers/azure/configs/services.py index 7bbaa1b13..793fb2ade 100644 --- a/ScoutSuite/providers/azure/configs/services.py +++ b/ScoutSuite/providers/azure/configs/services.py @@ -3,7 +3,7 @@ from ScoutSuite.providers.base.configs.services import BaseServicesConfig from ScoutSuite.providers.azure.services.storageaccounts import StorageAccountsConfig from ScoutSuite.providers.azure.services.monitor import MonitorConfig -from ScoutSuite.providers.azure.services.sqldatabase import SQLDatabaseConfig +from ScoutSuite.providers.azure.resources.sql.servers import ServersConfig as SQLDatabaseConfig from ScoutSuite.providers.azure.services.securitycenter import SecurityCenterConfig from ScoutSuite.providers.azure.services.network import NetworkConfig from ScoutSuite.providers.azure.services.keyvault import KeyVaultConfig @@ -31,7 +31,7 @@ def __init__(self, metadata=None, thread_config=4, **kwargs): self.storageaccounts = StorageAccountsConfig(thread_config=thread_config) self.monitor = MonitorConfig(thread_config=thread_config) - self.sqldatabase = SQLDatabaseConfig(thread_config=thread_config) + self.sqldatabase = SQLDatabaseConfig() self.securitycenter = SecurityCenterConfig(thread_config=thread_config) self.network = NetworkConfig(thread_config=thread_config) self.keyvault = KeyVaultConfig(thread_config=thread_config) From ba410c4e8a621758a4f3a9ffc20d2539f10f0fdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Wed, 20 Feb 2019 14:58:11 -0500 Subject: [PATCH 142/667] Remove useless keys and make children a list of tuples. --- .../providers/azure/resources/sql/databases.py | 14 +++++++------- .../providers/azure/resources/sql/servers.py | 10 +++++----- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/databases.py b/ScoutSuite/providers/azure/resources/sql/databases.py index 21ac1eb0d..732aabfdf 100644 --- a/ScoutSuite/providers/azure/resources/sql/databases.py +++ b/ScoutSuite/providers/azure/resources/sql/databases.py @@ -38,12 +38,12 @@ def _parse_transparent_data_encryption_config(config): } # register children resources: - children = { - 'Database Auditing Settings': (DatabaseBlobAuditingPoliciesConfig, _parse_auditing_config), - 'Database Threat Detection Policies': (DatabaseThreatDetectionPoliciesConfig, _parse_threat_detection_config), - 'Replication Links': (ReplicationLinksConfig, _parse_replication_links_config), - 'Transparent Data Encryptions': (TransparentDataEncryptionsConfig, _parse_transparent_data_encryption_config) - } + children = [ + (DatabaseBlobAuditingPoliciesConfig, _parse_auditing_config), + (DatabaseThreatDetectionPoliciesConfig, _parse_threat_detection_config), + (ReplicationLinksConfig, _parse_replication_links_config), + (TransparentDataEncryptionsConfig, _parse_transparent_data_encryption_config) + ] def __init__(self, resource_group_name, server_name): self.resource_group_name = resource_group_name @@ -62,7 +62,7 @@ async def fetch_all(self, credentials, regions=None, partition_name='aws', targe } # put the following code in a fetch_children() parent method (typical method for a composite node)? - for (resource_config, resource_parser) in self.children.values(): + for (resource_config, resource_parser) in self.children: resource = resource_config(self.resource_group_name, self.server_name, db.name) await resource.fetch_all(credentials) for k, v in resource_parser.__func__(resource).items(): diff --git a/ScoutSuite/providers/azure/resources/sql/servers.py b/ScoutSuite/providers/azure/resources/sql/servers.py index f1cc2b572..a44b913c9 100644 --- a/ScoutSuite/providers/azure/resources/sql/servers.py +++ b/ScoutSuite/providers/azure/resources/sql/servers.py @@ -26,10 +26,10 @@ def _parse_azure_ad_administrators_config(config): } # register children resources: - children = { - 'Databases': (DatabasesConfig, _parse_databases_config), - 'Azure AD Administrators': (ServerAzureAdAdministratorsConfig, _parse_azure_ad_administrators_config) - } + children = [ + (DatabasesConfig, _parse_databases_config), + (ServerAzureAdAdministratorsConfig, _parse_azure_ad_administrators_config) + ] def __init__(self): self.servers = {} @@ -51,7 +51,7 @@ async def fetch_all(self, credentials, regions=None, partition_name='aws', targe } # put the following code in a fetch_children() parent method (typical method for a composite node)? - for (resource_config, resource_parser) in self.children.values(): + for (resource_config, resource_parser) in self.children: resource = resource_config(resource_group_name, server.name) await resource.fetch_all(credentials) for k, v in resource_parser.__func__(resource).items(): From e3c36b60d3383f8fef866661b443c92fa0e6e21c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Wed, 20 Feb 2019 23:45:44 -0500 Subject: [PATCH 143/667] Modify base class and add parsing process. --- .../sql/database_blob_auditing_policies.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py b/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py index 1f1c1762a..743b0e8a2 100644 --- a/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py +++ b/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py @@ -2,22 +2,26 @@ from azure.mgmt.sql import SqlManagementClient -from ScoutSuite.providers.base.configs.resource_config import ResourceConfig +from ScoutSuite.providers.base.configs.resources import Resources -class DatabaseBlobAuditingPoliciesConfig(ResourceConfig): +class DatabaseBlobAuditingPolicies(Resources): def __init__(self, resource_group_name, server_name, database_name): self.resource_group_name = resource_group_name self.server_name = server_name self.database_name = database_name - self.value = None - - async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): + async def fetch_all(self, credentials): # sdk container: api = SqlManagementClient(credentials.credentials, credentials.subscription_id) - self.value =\ + policies =\ api.database_blob_auditing_policies.get(self.resource_group_name, self.server_name, self.database_name) # TODO: await api.database_blob_auditing_policies.get(self.resource_group_name, self.server_name, self.database_name) + + self['auditing_enabled'] = self._is_auditing_enabled(policies) + + @staticmethod + def _is_auditing_enabled(policies): + return policies.state == "Enabled" From 40b4ff2206e9dce78100922818fce862bb329edc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Wed, 20 Feb 2019 23:45:53 -0500 Subject: [PATCH 144/667] Modify base class and add parsing process. --- .../sql/database_threat_detection_policies.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py b/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py index 536bbd420..874b14307 100644 --- a/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py +++ b/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py @@ -2,22 +2,26 @@ from azure.mgmt.sql import SqlManagementClient -from ScoutSuite.providers.base.configs.resource_config import ResourceConfig +from ScoutSuite.providers.base.configs.resources import Resources -class DatabaseThreatDetectionPoliciesConfig(ResourceConfig): +class DatabaseThreatDetectionPolicies(Resources): def __init__(self, resource_group_name, server_name, database_name): self.resource_group_name = resource_group_name self.server_name = server_name self.database_name = database_name - self.value = None - - async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): + async def fetch_all(self, credentials): # sdk container: api = SqlManagementClient(credentials.credentials, credentials.subscription_id) - self.value =\ + policies =\ api.database_threat_detection_policies.get(self.resource_group_name, self.server_name, self.database_name) # TODO: await api.transparent_data_encryptions.get(self.resource_group_name, self.server_name, self.database_name) + + self['threat_detection_enabled'] = self._is_threat_detection_enabled(policies) + + @staticmethod + def _is_threat_detection_enabled(policies): + return policies.state == "Enabled" From 7c29f80bdea3123f174c77f7faff2c586ab6aa8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Wed, 20 Feb 2019 23:46:04 -0500 Subject: [PATCH 145/667] Modify base class and add parsing process. --- .../azure/resources/sql/replication_links.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/replication_links.py b/ScoutSuite/providers/azure/resources/sql/replication_links.py index 1005f0f17..20db831f9 100644 --- a/ScoutSuite/providers/azure/resources/sql/replication_links.py +++ b/ScoutSuite/providers/azure/resources/sql/replication_links.py @@ -2,22 +2,26 @@ from azure.mgmt.sql import SqlManagementClient -from ScoutSuite.providers.base.configs.resource_config import ResourceConfig +from ScoutSuite.providers.base.configs.resources import Resources -class ReplicationLinksConfig(ResourceConfig): +class ReplicationLinks(Resources): def __init__(self, resource_group_name, server_name, database_name): self.resource_group_name = resource_group_name self.server_name = server_name self.database_name = database_name - self.value = None - async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): # sdk container: api = SqlManagementClient(credentials.credentials, credentials.subscription_id) - self.value =\ + links =\ list(api.replication_links.list_by_database(self.resource_group_name, self.server_name, self.database_name)) # TODO: await api.replication_links.list_by_databases(self.resource_group_name, self.server_name, self.database_name) + + self['replication_configured'] = self._is_replication_configured(links) + + @staticmethod + def _is_replication_configured(links): + return len(links) > 0 From 9eaee5e57c98239a73d6088d36b2abbe2e1d6256 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Wed, 20 Feb 2019 23:46:16 -0500 Subject: [PATCH 146/667] Modify base class and add parsing process. --- .../resources/sql/server_azure_ad_administrators.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py b/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py index f68f91e9d..189a3cf2d 100644 --- a/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py +++ b/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py @@ -3,24 +3,23 @@ from azure.mgmt.sql import SqlManagementClient from msrestazure.azure_exceptions import CloudError -from ScoutSuite.providers.base.configs.resource_config import ResourceConfig +from ScoutSuite.providers.base.configs.resources import Resources -class ServerAzureAdAdministratorsConfig(ResourceConfig): +class ServerAzureAdAdministrators(Resources): def __init__(self, resource_group_name, server_name): self.resource_group_name = resource_group_name self.server_name = server_name - self.value = None - - async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): + async def fetch_all(self, credentials): # sdk container: api = SqlManagementClient(credentials.credentials, credentials.subscription_id) try: - self.value = \ + admins = \ api.server_azure_ad_administrators.get(self.resource_group_name, self.server_name) # TODO: await api.server_azure_ad_administrators.get(self.resource_group_name, self.server_name) + self['ad_admin_configured'] = True except CloudError: # no ad admin configured returns a 404 error - pass + self['ad_admin_configured'] = False From ff6825ae3e2b426499551328fb17b4771644bd3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Wed, 20 Feb 2019 23:46:25 -0500 Subject: [PATCH 147/667] Modify base class and add parsing process. --- .../resources/sql/transparent_data_encryptions.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py b/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py index e2a582046..95a971bf1 100644 --- a/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py +++ b/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py @@ -2,10 +2,10 @@ from azure.mgmt.sql import SqlManagementClient -from ScoutSuite.providers.base.configs.resource_config import ResourceConfig +from ScoutSuite.providers.base.configs.resources import Resources -class TransparentDataEncryptionsConfig(ResourceConfig): +class TransparentDataEncryptions(Resources): def __init__(self, resource_group_name, server_name, database_name): self.resource_group_name = resource_group_name @@ -18,7 +18,12 @@ async def fetch_all(self, credentials, regions=None, partition_name='aws', targe # sdk container: api = SqlManagementClient(credentials.credentials, credentials.subscription_id) - self.value =\ + encryptions =\ api.transparent_data_encryptions.get(self.resource_group_name, self.server_name, self.database_name) # TODO: await api.transparent_data_encryptions.get(self.resource_group_name, self.server_name, self.database_name) + self['transparent_data_encryption_enabled'] = self._is_transparent_data_encryption_enabled(encryptions) + + @staticmethod + def _is_transparent_data_encryption_enabled(encryptions): + return encryptions.status == "Enabled" From f97cc9ccda3c47781bf358bd9187dd12c093e682 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Wed, 20 Feb 2019 23:46:55 -0500 Subject: [PATCH 148/667] Modify base class and remove parsing processes of children. --- .../azure/resources/sql/databases.py | 63 +++++-------------- 1 file changed, 17 insertions(+), 46 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/databases.py b/ScoutSuite/providers/azure/resources/sql/databases.py index 732aabfdf..c58891f6e 100644 --- a/ScoutSuite/providers/azure/resources/sql/databases.py +++ b/ScoutSuite/providers/azure/resources/sql/databases.py @@ -2,68 +2,39 @@ from azure.mgmt.sql import SqlManagementClient -from ScoutSuite.providers.base.configs.resource_config import ResourceConfig +from ScoutSuite.providers.base.configs.resources import Resources -from .database_blob_auditing_policies import DatabaseBlobAuditingPoliciesConfig -from .database_threat_detection_policies import DatabaseThreatDetectionPoliciesConfig -from .replication_links import ReplicationLinksConfig -from .transparent_data_encryptions import TransparentDataEncryptionsConfig +from .database_blob_auditing_policies import DatabaseBlobAuditingPolicies +from .database_threat_detection_policies import DatabaseThreatDetectionPolicies +from .replication_links import ReplicationLinks +from .transparent_data_encryptions import TransparentDataEncryptions -class DatabasesConfig(ResourceConfig): - # the following static methods will be used to parse the config of each resource nested in a database config - # (defined in 'children' attribute below): - @staticmethod - def _parse_auditing_config(config): - return { - 'auditing_enabled': config.value.state == "Enabled" - } - - @staticmethod - def _parse_threat_detection_config(config): - return { - 'threat_detection_enabled': config.value.state == "Enabled" - } - - @staticmethod - def _parse_replication_links_config(config): - return { - 'replication_configured': len(config.value) > 0 - } - - @staticmethod - def _parse_transparent_data_encryption_config(config): - return { - 'transparent_data_encryption_enabled': config.value.status == "Enabled" - } - - # register children resources: +class Databases(Resources): children = [ - (DatabaseBlobAuditingPoliciesConfig, _parse_auditing_config), - (DatabaseThreatDetectionPoliciesConfig, _parse_threat_detection_config), - (ReplicationLinksConfig, _parse_replication_links_config), - (TransparentDataEncryptionsConfig, _parse_transparent_data_encryption_config) + DatabaseBlobAuditingPolicies, + DatabaseThreatDetectionPolicies, + ReplicationLinks, + TransparentDataEncryptions, ] def __init__(self, resource_group_name, server_name): self.resource_group_name = resource_group_name self.server_name = server_name - self.databases = {} - - async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): + async def fetch_all(self, credentials): # sdk container: api = SqlManagementClient(credentials.credentials, credentials.subscription_id) + self['databases'] = {} # TODO: for db in await api.databases.list_by_server(self.resource_group_name, self.server_name): for db in api.databases.list_by_server(self.resource_group_name, self.server_name): - self.databases[db.name] = { + self['databases'][db.name] = { 'id': db.name, } # put the following code in a fetch_children() parent method (typical method for a composite node)? - for (resource_config, resource_parser) in self.children: - resource = resource_config(self.resource_group_name, self.server_name, db.name) - await resource.fetch_all(credentials) - for k, v in resource_parser.__func__(resource).items(): - self.databases[db.name][k] = v + for resources_class in self.simple_children: + resources = resources_class(self.resource_group_name, self.server_name, db.name) + await resources.fetch_all(credentials) + self['databases'][db.name].update(resources) From a45f74776c412336cef7dbe9aaad1e45ee6117bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Wed, 20 Feb 2019 23:47:07 -0500 Subject: [PATCH 149/667] Modify base class and remove parsing processes of children. --- .../providers/azure/resources/sql/servers.py | 48 ++++++------------- 1 file changed, 14 insertions(+), 34 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/servers.py b/ScoutSuite/providers/azure/resources/sql/servers.py index a44b913c9..48b5de91b 100644 --- a/ScoutSuite/providers/azure/resources/sql/servers.py +++ b/ScoutSuite/providers/azure/resources/sql/servers.py @@ -2,57 +2,37 @@ from azure.mgmt.sql import SqlManagementClient -from ScoutSuite.providers.base.configs.resource_config import ResourceConfig +from ScoutSuite.providers.base.configs.resources import Resources from ScoutSuite.providers.azure.utils import get_resource_group_name -from .databases import DatabasesConfig -from .server_azure_ad_administrators import ServerAzureAdAdministratorsConfig +from .databases import Databases +from .server_azure_ad_administrators import ServerAzureAdAdministrators from ..utils import get_non_provider_id -class ServersConfig(ResourceConfig): - # the following static methods will be used to parse the config of each resource nested in a server config - # (defined in 'children' attribute below): - @staticmethod - def _parse_databases_config(config): - return { - 'databases': config.databases - } - - @staticmethod - def _parse_azure_ad_administrators_config(config): - return { - 'ad_admin_configured': config.value is not None - } - - # register children resources: +class Servers(Resources): children = [ - (DatabasesConfig, _parse_databases_config), - (ServerAzureAdAdministratorsConfig, _parse_azure_ad_administrators_config) + Databases, + ServerAzureAdAdministrators, ] - def __init__(self): - self.servers = {} - - async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): + async def fetch_all(self, credentials): # sdk container: api = SqlManagementClient(credentials.credentials, credentials.subscription_id) - # for server in await api.servers.list(): + self['servers'] = {} + # TODO: for server in await api.servers.list(): for server in api.servers.list(): - # parsing: - id = get_non_provider_id(server.id) resource_group_name = get_resource_group_name(server.id) - self.servers[id] = { + self['servers'][id] = { 'id': id, 'name': server.name } # put the following code in a fetch_children() parent method (typical method for a composite node)? - for (resource_config, resource_parser) in self.children: - resource = resource_config(resource_group_name, server.name) - await resource.fetch_all(credentials) - for k, v in resource_parser.__func__(resource).items(): - self.servers[id][k] = v + for resources_class in self.children: + resources = resources_class(resource_group_name, server.name) + await resources.fetch_all(credentials) + self['servers'][id].update(resources) From 2db4ce2dde290f0b2822837ded89a0f33dbf44e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 16:12:34 -0500 Subject: [PATCH 150/667] Add AzureSimpleResources and AzureCompositeResources abstract classes. --- .../providers/azure/resources/resources.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 ScoutSuite/providers/azure/resources/resources.py diff --git a/ScoutSuite/providers/azure/resources/resources.py b/ScoutSuite/providers/azure/resources/resources.py new file mode 100644 index 000000000..36072503e --- /dev/null +++ b/ScoutSuite/providers/azure/resources/resources.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- + +import abc + +from ScoutSuite.providers.base.configs.resources import SimpleResources +from ScoutSuite.providers.base.configs.resources import CompositeResources + + +class AzureSimpleResources(SimpleResources, metaclass=abc.ABCMeta): + + async def fetch_all(self, credentials): + raw_resource = await self.get_resources_from_api(credentials) + name, resource = self.parse_resource(raw_resource) + self[name] = resource + + +class AzureCompositeResources(CompositeResources, metaclass=abc.ABCMeta): + + # TODO: get rid of the credentials. + async def fetch_children(self, parent, credentials, **kwargs): + for child_class in self.children: + child = child_class(**kwargs) + await child.fetch_all(credentials) + parent.update(child) From d558ec06ff938b3178ba4f0b93f9d4577aa9c78b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 16:13:27 -0500 Subject: [PATCH 151/667] Make DatabaseBlobAuditingPolicies AzureSimpleResources. --- .../sql/database_blob_auditing_policies.py | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py b/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py index 743b0e8a2..b2d3d74a2 100644 --- a/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py +++ b/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py @@ -2,26 +2,21 @@ from azure.mgmt.sql import SqlManagementClient -from ScoutSuite.providers.base.configs.resources import Resources +from ..resources import AzureSimpleResources -class DatabaseBlobAuditingPolicies(Resources): +class DatabaseBlobAuditingPolicies(AzureSimpleResources): def __init__(self, resource_group_name, server_name, database_name): self.resource_group_name = resource_group_name self.server_name = server_name self.database_name = database_name - async def fetch_all(self, credentials): - # sdk container: + # TODO: make it really async. + async def get_resources_from_api(self, credentials): api = SqlManagementClient(credentials.credentials, credentials.subscription_id) + return api.database_blob_auditing_policies.get( + self.resource_group_name, self.server_name, self.database_name) - policies =\ - api.database_blob_auditing_policies.get(self.resource_group_name, self.server_name, self.database_name) - # TODO: await api.database_blob_auditing_policies.get(self.resource_group_name, self.server_name, self.database_name) - - self['auditing_enabled'] = self._is_auditing_enabled(policies) - - @staticmethod - def _is_auditing_enabled(policies): - return policies.state == "Enabled" + def parse_resource(self, policies): + return 'auditing_enabled', policies.state == "Enabled" From ecf5db9bb35864c6a7f06f8ca732baaad4d500cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 16:13:50 -0500 Subject: [PATCH 152/667] Make DatabaseThreatDetectionPolicies AzureSimpleResources. --- .../sql/database_threat_detection_policies.py | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py b/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py index 874b14307..2563f813a 100644 --- a/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py +++ b/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py @@ -2,26 +2,21 @@ from azure.mgmt.sql import SqlManagementClient -from ScoutSuite.providers.base.configs.resources import Resources +from ..resources import AzureSimpleResources -class DatabaseThreatDetectionPolicies(Resources): +class DatabaseThreatDetectionPolicies(AzureSimpleResources): def __init__(self, resource_group_name, server_name, database_name): self.resource_group_name = resource_group_name self.server_name = server_name self.database_name = database_name - async def fetch_all(self, credentials): - # sdk container: + # TODO: make it really async. + async def get_resources_from_api(self, credentials): api = SqlManagementClient(credentials.credentials, credentials.subscription_id) + return api.database_threat_detection_policies.get( + self.resource_group_name, self.server_name, self.database_name) - policies =\ - api.database_threat_detection_policies.get(self.resource_group_name, self.server_name, self.database_name) - # TODO: await api.transparent_data_encryptions.get(self.resource_group_name, self.server_name, self.database_name) - - self['threat_detection_enabled'] = self._is_threat_detection_enabled(policies) - - @staticmethod - def _is_threat_detection_enabled(policies): - return policies.state == "Enabled" + def parse_resource(self, policies): + return 'threat_detection_enabled', policies.state == "Enabled" From 360245eecd8c7b12c5e54e30260f68e25f2b8de3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 16:14:11 -0500 Subject: [PATCH 153/667] Make ReplicationLinks AzureSimpleResources. --- .../azure/resources/sql/replication_links.py | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/replication_links.py b/ScoutSuite/providers/azure/resources/sql/replication_links.py index 20db831f9..dbedd953d 100644 --- a/ScoutSuite/providers/azure/resources/sql/replication_links.py +++ b/ScoutSuite/providers/azure/resources/sql/replication_links.py @@ -2,26 +2,21 @@ from azure.mgmt.sql import SqlManagementClient -from ScoutSuite.providers.base.configs.resources import Resources +from ..resources import AzureSimpleResources -class ReplicationLinks(Resources): +class ReplicationLinks(AzureSimpleResources): def __init__(self, resource_group_name, server_name, database_name): self.resource_group_name = resource_group_name self.server_name = server_name self.database_name = database_name - async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): - # sdk container: + # TODO: make it really async. + async def get_resources_from_api(self, credentials): api = SqlManagementClient(credentials.credentials, credentials.subscription_id) + return list(api.replication_links.list_by_database( + self.resource_group_name, self.server_name, self.database_name)) - links =\ - list(api.replication_links.list_by_database(self.resource_group_name, self.server_name, self.database_name)) - # TODO: await api.replication_links.list_by_databases(self.resource_group_name, self.server_name, self.database_name) - - self['replication_configured'] = self._is_replication_configured(links) - - @staticmethod - def _is_replication_configured(links): - return len(links) > 0 + def parse_resource(self, links): + return 'replication_configured', len(links) > 0 From b452c9102cb2ba82cf4d38fc275a9f41e14ef0d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 16:15:08 -0500 Subject: [PATCH 154/667] Remove unused variable. --- .../azure/resources/sql/server_azure_ad_administrators.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py b/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py index 189a3cf2d..a6de8eba7 100644 --- a/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py +++ b/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py @@ -12,14 +12,13 @@ def __init__(self, resource_group_name, server_name): self.resource_group_name = resource_group_name self.server_name = server_name + # TODO: make it really async. async def fetch_all(self, credentials): # sdk container: api = SqlManagementClient(credentials.credentials, credentials.subscription_id) try: - admins = \ - api.server_azure_ad_administrators.get(self.resource_group_name, self.server_name) - # TODO: await api.server_azure_ad_administrators.get(self.resource_group_name, self.server_name) + _ = api.server_azure_ad_administrators.get(self.resource_group_name, self.server_name) self['ad_admin_configured'] = True except CloudError: # no ad admin configured returns a 404 error self['ad_admin_configured'] = False From 7e31e08bc49a91f035be44db66310af88c7c49da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 16:15:33 -0500 Subject: [PATCH 155/667] Make TransparentDataEncryptions AzureSimpleResources. --- .../sql/transparent_data_encryptions.py | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py b/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py index 95a971bf1..becc2748c 100644 --- a/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py +++ b/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py @@ -2,28 +2,21 @@ from azure.mgmt.sql import SqlManagementClient -from ScoutSuite.providers.base.configs.resources import Resources +from ..resources import AzureSimpleResources -class TransparentDataEncryptions(Resources): +class TransparentDataEncryptions(AzureSimpleResources): def __init__(self, resource_group_name, server_name, database_name): self.resource_group_name = resource_group_name self.server_name = server_name self.database_name = database_name - self.value = None - - async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): - # sdk container: + # TODO: make it really async. + async def get_resources_from_api(self, credentials): api = SqlManagementClient(credentials.credentials, credentials.subscription_id) + return api.transparent_data_encryptions.get( + self.resource_group_name, self.server_name, self.database_name) - encryptions =\ - api.transparent_data_encryptions.get(self.resource_group_name, self.server_name, self.database_name) - # TODO: await api.transparent_data_encryptions.get(self.resource_group_name, self.server_name, self.database_name) - - self['transparent_data_encryption_enabled'] = self._is_transparent_data_encryption_enabled(encryptions) - - @staticmethod - def _is_transparent_data_encryption_enabled(encryptions): - return encryptions.status == "Enabled" + def parse_resource(self, encryptions): + return 'transparent_data_encryption_enabled', encryptions.status == "Enabled" From cdfd6e5daf23e893280a430dab60d390a5abd138 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 16:16:08 -0500 Subject: [PATCH 156/667] Make Databases AzureCompositeResources. --- .../azure/resources/sql/databases.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/databases.py b/ScoutSuite/providers/azure/resources/sql/databases.py index c58891f6e..8178ff3be 100644 --- a/ScoutSuite/providers/azure/resources/sql/databases.py +++ b/ScoutSuite/providers/azure/resources/sql/databases.py @@ -2,7 +2,7 @@ from azure.mgmt.sql import SqlManagementClient -from ScoutSuite.providers.base.configs.resources import Resources +from ..resources import AzureCompositeResources from .database_blob_auditing_policies import DatabaseBlobAuditingPolicies from .database_threat_detection_policies import DatabaseThreatDetectionPolicies @@ -10,7 +10,7 @@ from .transparent_data_encryptions import TransparentDataEncryptions -class Databases(Resources): +class Databases(AzureCompositeResources): children = [ DatabaseBlobAuditingPolicies, DatabaseThreatDetectionPolicies, @@ -22,19 +22,19 @@ def __init__(self, resource_group_name, server_name): self.resource_group_name = resource_group_name self.server_name = server_name + # TODO: make it really async. async def fetch_all(self, credentials): - # sdk container: api = SqlManagementClient(credentials.credentials, credentials.subscription_id) self['databases'] = {} - # TODO: for db in await api.databases.list_by_server(self.resource_group_name, self.server_name): for db in api.databases.list_by_server(self.resource_group_name, self.server_name): self['databases'][db.name] = { 'id': db.name, } - - # put the following code in a fetch_children() parent method (typical method for a composite node)? - for resources_class in self.simple_children: - resources = resources_class(self.resource_group_name, self.server_name, db.name) - await resources.fetch_all(credentials) - self['databases'][db.name].update(resources) + await self.fetch_children( + parent=self['databases'][db.name], + resource_group_name=self.resource_group_name, + server_name=self.server_name, + database_name=db.name, + credentials=credentials + ) From 1bf9d813184b5118b432c630bb708ef84cd21de4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 16:16:30 -0500 Subject: [PATCH 157/667] Make Servers AzureCompositeResources. --- .../providers/azure/resources/sql/servers.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/servers.py b/ScoutSuite/providers/azure/resources/sql/servers.py index 48b5de91b..9eab303c2 100644 --- a/ScoutSuite/providers/azure/resources/sql/servers.py +++ b/ScoutSuite/providers/azure/resources/sql/servers.py @@ -2,7 +2,7 @@ from azure.mgmt.sql import SqlManagementClient -from ScoutSuite.providers.base.configs.resources import Resources +from ..resources import AzureCompositeResources from ScoutSuite.providers.azure.utils import get_resource_group_name from .databases import Databases @@ -10,18 +10,18 @@ from ..utils import get_non_provider_id -class Servers(Resources): +class Servers(AzureCompositeResources): children = [ Databases, ServerAzureAdAdministrators, ] - async def fetch_all(self, credentials): + # TODO: make it really async. + async def fetch_all(self, credentials, **kwargs): # sdk container: api = SqlManagementClient(credentials.credentials, credentials.subscription_id) self['servers'] = {} - # TODO: for server in await api.servers.list(): for server in api.servers.list(): id = get_non_provider_id(server.id) resource_group_name = get_resource_group_name(server.id) @@ -30,9 +30,10 @@ async def fetch_all(self, credentials): 'id': id, 'name': server.name } + await self.fetch_children( + parent=self['servers'][id], + resource_group_name=resource_group_name, + server_name=server.name, + credentials=credentials) - # put the following code in a fetch_children() parent method (typical method for a composite node)? - for resources_class in self.children: - resources = resources_class(resource_group_name, server.name) - await resources.fetch_all(credentials) - self['servers'][id].update(resources) + self['servers_count'] = len(self['servers']) From f26246ebda1a594c16f49669410e0f7d7e029732 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 16:16:49 -0500 Subject: [PATCH 158/667] Update services. --- ScoutSuite/providers/azure/configs/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/azure/configs/services.py b/ScoutSuite/providers/azure/configs/services.py index 793fb2ade..c5520dcfd 100644 --- a/ScoutSuite/providers/azure/configs/services.py +++ b/ScoutSuite/providers/azure/configs/services.py @@ -3,7 +3,7 @@ from ScoutSuite.providers.base.configs.services import BaseServicesConfig from ScoutSuite.providers.azure.services.storageaccounts import StorageAccountsConfig from ScoutSuite.providers.azure.services.monitor import MonitorConfig -from ScoutSuite.providers.azure.resources.sql.servers import ServersConfig as SQLDatabaseConfig +from ScoutSuite.providers.azure.resources.sql.servers import Servers as SQLDatabaseConfig from ScoutSuite.providers.azure.services.securitycenter import SecurityCenterConfig from ScoutSuite.providers.azure.services.network import NetworkConfig from ScoutSuite.providers.azure.services.keyvault import KeyVaultConfig From 097e9c5c390329bda0b3bbbfb25fb26181fb3ae6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 16:36:03 -0500 Subject: [PATCH 159/667] Use Azure API through a facade. --- .../resources/sql/database_blob_auditing_policies.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py b/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py index b2d3d74a2..1f617ca4d 100644 --- a/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py +++ b/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py @@ -7,15 +7,15 @@ class DatabaseBlobAuditingPolicies(AzureSimpleResources): - def __init__(self, resource_group_name, server_name, database_name): + def __init__(self, resource_group_name, server_name, database_name, facade): self.resource_group_name = resource_group_name self.server_name = server_name self.database_name = database_name + self.facade = facade # TODO: make it really async. - async def get_resources_from_api(self, credentials): - api = SqlManagementClient(credentials.credentials, credentials.subscription_id) - return api.database_blob_auditing_policies.get( + async def get_resources_from_api(self): + return self.facade.database_blob_auditing_policies.get( self.resource_group_name, self.server_name, self.database_name) def parse_resource(self, policies): From 051f87d712879cffc615ab4ff954deb9c3f61787 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 16:36:26 -0500 Subject: [PATCH 160/667] Use Azure API through a facade. --- .../resources/sql/database_threat_detection_policies.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py b/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py index 2563f813a..61dc696fd 100644 --- a/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py +++ b/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py @@ -7,15 +7,15 @@ class DatabaseThreatDetectionPolicies(AzureSimpleResources): - def __init__(self, resource_group_name, server_name, database_name): + def __init__(self, resource_group_name, server_name, database_name, facade): self.resource_group_name = resource_group_name self.server_name = server_name self.database_name = database_name + self.facade = facade # TODO: make it really async. - async def get_resources_from_api(self, credentials): - api = SqlManagementClient(credentials.credentials, credentials.subscription_id) - return api.database_threat_detection_policies.get( + async def get_resources_from_api(self): + return self.facade.database_threat_detection_policies.get( self.resource_group_name, self.server_name, self.database_name) def parse_resource(self, policies): From da90b53890ac675eab68cbf128700df0fe758f04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 16:36:43 -0500 Subject: [PATCH 161/667] Use Azure API through a facade. --- .../providers/azure/resources/sql/replication_links.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/replication_links.py b/ScoutSuite/providers/azure/resources/sql/replication_links.py index dbedd953d..aca33b763 100644 --- a/ScoutSuite/providers/azure/resources/sql/replication_links.py +++ b/ScoutSuite/providers/azure/resources/sql/replication_links.py @@ -7,15 +7,15 @@ class ReplicationLinks(AzureSimpleResources): - def __init__(self, resource_group_name, server_name, database_name): + def __init__(self, resource_group_name, server_name, database_name, facade): self.resource_group_name = resource_group_name self.server_name = server_name self.database_name = database_name + self.facade = facade # TODO: make it really async. - async def get_resources_from_api(self, credentials): - api = SqlManagementClient(credentials.credentials, credentials.subscription_id) - return list(api.replication_links.list_by_database( + async def get_resources_from_api(self): + return list(self.facade.replication_links.list_by_database( self.resource_group_name, self.server_name, self.database_name)) def parse_resource(self, links): From 84a1bb08009f2accc895c2fd7b1faa21f2881ee5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 16:36:56 -0500 Subject: [PATCH 162/667] Use Azure API through a facade. --- .../resources/sql/server_azure_ad_administrators.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py b/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py index a6de8eba7..1fe551488 100644 --- a/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py +++ b/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py @@ -8,17 +8,15 @@ class ServerAzureAdAdministrators(Resources): - def __init__(self, resource_group_name, server_name): + def __init__(self, resource_group_name, server_name, facade): self.resource_group_name = resource_group_name self.server_name = server_name + self.facade = facade # TODO: make it really async. - async def fetch_all(self, credentials): - # sdk container: - api = SqlManagementClient(credentials.credentials, credentials.subscription_id) - + async def fetch_all(self): try: - _ = api.server_azure_ad_administrators.get(self.resource_group_name, self.server_name) + _ = self.facade.server_azure_ad_administrators.get(self.resource_group_name, self.server_name) self['ad_admin_configured'] = True except CloudError: # no ad admin configured returns a 404 error self['ad_admin_configured'] = False From bab8f014e29c39677f12ae66f2aa76aa443eba8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 16:37:07 -0500 Subject: [PATCH 163/667] Use Azure API through a facade. --- .../azure/resources/sql/transparent_data_encryptions.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py b/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py index becc2748c..e52f6e3df 100644 --- a/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py +++ b/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py @@ -7,15 +7,15 @@ class TransparentDataEncryptions(AzureSimpleResources): - def __init__(self, resource_group_name, server_name, database_name): + def __init__(self, resource_group_name, server_name, database_name, facade): self.resource_group_name = resource_group_name self.server_name = server_name self.database_name = database_name + self.facade = facade # TODO: make it really async. - async def get_resources_from_api(self, credentials): - api = SqlManagementClient(credentials.credentials, credentials.subscription_id) - return api.transparent_data_encryptions.get( + async def get_resources_from_api(self): + return self.facade.transparent_data_encryptions.get( self.resource_group_name, self.server_name, self.database_name) def parse_resource(self, encryptions): From e13119920ccafd376bc182e26ac43f66b4fd097a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 16:37:30 -0500 Subject: [PATCH 164/667] Use Azure API through a facade and ignore master db. --- .../providers/azure/resources/sql/databases.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/databases.py b/ScoutSuite/providers/azure/resources/sql/databases.py index 8178ff3be..9a37673f7 100644 --- a/ScoutSuite/providers/azure/resources/sql/databases.py +++ b/ScoutSuite/providers/azure/resources/sql/databases.py @@ -18,16 +18,20 @@ class Databases(AzureCompositeResources): TransparentDataEncryptions, ] - def __init__(self, resource_group_name, server_name): + def __init__(self, resource_group_name, server_name, facade): self.resource_group_name = resource_group_name self.server_name = server_name + self.facade = facade # TODO: make it really async. - async def fetch_all(self, credentials): - api = SqlManagementClient(credentials.credentials, credentials.subscription_id) - + async def fetch_all(self): self['databases'] = {} - for db in api.databases.list_by_server(self.resource_group_name, self.server_name): + + for db in self.facade.databases.list_by_server(self.resource_group_name, self.server_name): + # We do not want to scan 'master' database which is auto-generated by Azure and read-only: + if db.name == 'master': + continue + self['databases'][db.name] = { 'id': db.name, } @@ -36,5 +40,5 @@ async def fetch_all(self, credentials): resource_group_name=self.resource_group_name, server_name=self.server_name, database_name=db.name, - credentials=credentials + facade=self.facade ) From 71d07f82c67e2e428f7644bf77a948662dde5eda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 16:39:54 -0500 Subject: [PATCH 165/667] Define a facade for using Azure API. --- ScoutSuite/providers/azure/resources/sql/servers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/servers.py b/ScoutSuite/providers/azure/resources/sql/servers.py index 9eab303c2..9c268f95e 100644 --- a/ScoutSuite/providers/azure/resources/sql/servers.py +++ b/ScoutSuite/providers/azure/resources/sql/servers.py @@ -18,11 +18,11 @@ class Servers(AzureCompositeResources): # TODO: make it really async. async def fetch_all(self, credentials, **kwargs): - # sdk container: - api = SqlManagementClient(credentials.credentials, credentials.subscription_id) + # TODO: build that facade somewhere else: + facade = SqlManagementClient(credentials.credentials, credentials.subscription_id) self['servers'] = {} - for server in api.servers.list(): + for server in facade.servers.list(): id = get_non_provider_id(server.id) resource_group_name = get_resource_group_name(server.id) @@ -34,6 +34,6 @@ async def fetch_all(self, credentials, **kwargs): parent=self['servers'][id], resource_group_name=resource_group_name, server_name=server.name, - credentials=credentials) + facade=facade) self['servers_count'] = len(self['servers']) From c5924601a6e39e59cf6b8ba1d468911136190530 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 16:40:16 -0500 Subject: [PATCH 166/667] Remove credentials argument. --- ScoutSuite/providers/azure/resources/resources.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/resources.py b/ScoutSuite/providers/azure/resources/resources.py index 36072503e..02a8d6097 100644 --- a/ScoutSuite/providers/azure/resources/resources.py +++ b/ScoutSuite/providers/azure/resources/resources.py @@ -8,8 +8,8 @@ class AzureSimpleResources(SimpleResources, metaclass=abc.ABCMeta): - async def fetch_all(self, credentials): - raw_resource = await self.get_resources_from_api(credentials) + async def fetch_all(self): + raw_resource = await self.get_resources_from_api() name, resource = self.parse_resource(raw_resource) self[name] = resource @@ -17,8 +17,8 @@ async def fetch_all(self, credentials): class AzureCompositeResources(CompositeResources, metaclass=abc.ABCMeta): # TODO: get rid of the credentials. - async def fetch_children(self, parent, credentials, **kwargs): + async def fetch_children(self, parent, **kwargs): for child_class in self.children: child = child_class(**kwargs) - await child.fetch_all(credentials) + await child.fetch_all() parent.update(child) From 10dd0589df166ff138cf0e76d4a0adfac7feeb9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 24 Feb 2019 22:41:50 -0500 Subject: [PATCH 167/667] Remove useless variable. --- .../azure/resources/sql/server_azure_ad_administrators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py b/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py index 1fe551488..778f774e1 100644 --- a/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py +++ b/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py @@ -16,7 +16,7 @@ def __init__(self, resource_group_name, server_name, facade): # TODO: make it really async. async def fetch_all(self): try: - _ = self.facade.server_azure_ad_administrators.get(self.resource_group_name, self.server_name) + self.facade.server_azure_ad_administrators.get(self.resource_group_name, self.server_name) self['ad_admin_configured'] = True except CloudError: # no ad admin configured returns a 404 error self['ad_admin_configured'] = False From 3f1ac40327aef9a70f9f3ad0e674b4381bf7d04b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Mon, 25 Feb 2019 10:21:54 -0500 Subject: [PATCH 168/667] Remove unused import statement. --- .../azure/resources/sql/database_blob_auditing_policies.py | 2 -- .../azure/resources/sql/database_threat_detection_policies.py | 2 -- ScoutSuite/providers/azure/resources/sql/databases.py | 2 -- ScoutSuite/providers/azure/resources/sql/replication_links.py | 2 -- .../azure/resources/sql/server_azure_ad_administrators.py | 1 - .../azure/resources/sql/transparent_data_encryptions.py | 2 -- 6 files changed, 11 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py b/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py index 1f617ca4d..ea2f4c116 100644 --- a/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py +++ b/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- -from azure.mgmt.sql import SqlManagementClient - from ..resources import AzureSimpleResources diff --git a/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py b/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py index 61dc696fd..4f260a3ee 100644 --- a/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py +++ b/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- -from azure.mgmt.sql import SqlManagementClient - from ..resources import AzureSimpleResources diff --git a/ScoutSuite/providers/azure/resources/sql/databases.py b/ScoutSuite/providers/azure/resources/sql/databases.py index 9a37673f7..4de894a47 100644 --- a/ScoutSuite/providers/azure/resources/sql/databases.py +++ b/ScoutSuite/providers/azure/resources/sql/databases.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- -from azure.mgmt.sql import SqlManagementClient - from ..resources import AzureCompositeResources from .database_blob_auditing_policies import DatabaseBlobAuditingPolicies diff --git a/ScoutSuite/providers/azure/resources/sql/replication_links.py b/ScoutSuite/providers/azure/resources/sql/replication_links.py index aca33b763..837988a50 100644 --- a/ScoutSuite/providers/azure/resources/sql/replication_links.py +++ b/ScoutSuite/providers/azure/resources/sql/replication_links.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- -from azure.mgmt.sql import SqlManagementClient - from ..resources import AzureSimpleResources diff --git a/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py b/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py index 778f774e1..7666757e0 100644 --- a/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py +++ b/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from azure.mgmt.sql import SqlManagementClient from msrestazure.azure_exceptions import CloudError from ScoutSuite.providers.base.configs.resources import Resources diff --git a/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py b/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py index e52f6e3df..4ebc04c89 100644 --- a/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py +++ b/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- -from azure.mgmt.sql import SqlManagementClient - from ..resources import AzureSimpleResources From acc4618a7925619606784d7c0e7327593145a742 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Mon, 25 Feb 2019 14:25:35 -0500 Subject: [PATCH 169/667] Make parse_resource method update its internal dict instead of returning a tuple. --- ScoutSuite/providers/azure/resources/resources.py | 3 +-- .../azure/resources/sql/database_blob_auditing_policies.py | 4 +++- .../azure/resources/sql/database_threat_detection_policies.py | 4 +++- ScoutSuite/providers/azure/resources/sql/replication_links.py | 4 +++- .../azure/resources/sql/transparent_data_encryptions.py | 4 +++- 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/resources.py b/ScoutSuite/providers/azure/resources/resources.py index 02a8d6097..2749ec9ff 100644 --- a/ScoutSuite/providers/azure/resources/resources.py +++ b/ScoutSuite/providers/azure/resources/resources.py @@ -10,8 +10,7 @@ class AzureSimpleResources(SimpleResources, metaclass=abc.ABCMeta): async def fetch_all(self): raw_resource = await self.get_resources_from_api() - name, resource = self.parse_resource(raw_resource) - self[name] = resource + self.parse_resource(raw_resource) class AzureCompositeResources(CompositeResources, metaclass=abc.ABCMeta): diff --git a/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py b/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py index ea2f4c116..637eae5dd 100644 --- a/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py +++ b/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py @@ -17,4 +17,6 @@ async def get_resources_from_api(self): self.resource_group_name, self.server_name, self.database_name) def parse_resource(self, policies): - return 'auditing_enabled', policies.state == "Enabled" + self.update({ + 'auditing_enabled': policies.state == "Enabled" + }) diff --git a/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py b/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py index 4f260a3ee..84977378f 100644 --- a/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py +++ b/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py @@ -17,4 +17,6 @@ async def get_resources_from_api(self): self.resource_group_name, self.server_name, self.database_name) def parse_resource(self, policies): - return 'threat_detection_enabled', policies.state == "Enabled" + self.update({ + 'threat_detection_enabled': policies.state == "Enabled" + }) diff --git a/ScoutSuite/providers/azure/resources/sql/replication_links.py b/ScoutSuite/providers/azure/resources/sql/replication_links.py index 837988a50..f15e7e27c 100644 --- a/ScoutSuite/providers/azure/resources/sql/replication_links.py +++ b/ScoutSuite/providers/azure/resources/sql/replication_links.py @@ -17,4 +17,6 @@ async def get_resources_from_api(self): self.resource_group_name, self.server_name, self.database_name)) def parse_resource(self, links): - return 'replication_configured', len(links) > 0 + self.update({ + 'replication_configured': len(links) > 0 + }) diff --git a/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py b/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py index 4ebc04c89..d82b51fce 100644 --- a/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py +++ b/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py @@ -17,4 +17,6 @@ async def get_resources_from_api(self): self.resource_group_name, self.server_name, self.database_name) def parse_resource(self, encryptions): - return 'transparent_data_encryption_enabled', encryptions.status == "Enabled" + return self.update({ + 'transparent_data_encryption_enabled': encryptions.status == "Enabled" + }) From 57a53fac654359ea74e9644fa00d634a33fe90bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 3 Mar 2019 12:36:31 -0500 Subject: [PATCH 170/667] Get rid of AzureSimpleResources and update AzureCompositeResources. --- ScoutSuite/providers/azure/resources/resources.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/resources.py b/ScoutSuite/providers/azure/resources/resources.py index 2749ec9ff..c802dc13a 100644 --- a/ScoutSuite/providers/azure/resources/resources.py +++ b/ScoutSuite/providers/azure/resources/resources.py @@ -2,22 +2,13 @@ import abc -from ScoutSuite.providers.base.configs.resources import SimpleResources from ScoutSuite.providers.base.configs.resources import CompositeResources -class AzureSimpleResources(SimpleResources, metaclass=abc.ABCMeta): - - async def fetch_all(self): - raw_resource = await self.get_resources_from_api() - self.parse_resource(raw_resource) - - class AzureCompositeResources(CompositeResources, metaclass=abc.ABCMeta): - # TODO: get rid of the credentials. - async def fetch_children(self, parent, **kwargs): - for child_class in self.children: + async def _fetch_children(self, parent, **kwargs): + for child_class in self._children: child = child_class(**kwargs) await child.fetch_all() parent.update(child) From 68b218b0337110c349a52b93ddb4988d639a3c92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 3 Mar 2019 12:37:08 -0500 Subject: [PATCH 171/667] Update database_blob_auditing_policies.py --- .../resources/sql/database_blob_auditing_policies.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py b/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py index 637eae5dd..2e0b2f66f 100644 --- a/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py +++ b/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- -from ..resources import AzureSimpleResources +from ScoutSuite.providers.base.configs.resources import Resources -class DatabaseBlobAuditingPolicies(AzureSimpleResources): +class DatabaseBlobAuditingPolicies(Resources): def __init__(self, resource_group_name, server_name, database_name, facade): self.resource_group_name = resource_group_name @@ -12,11 +12,12 @@ def __init__(self, resource_group_name, server_name, database_name, facade): self.facade = facade # TODO: make it really async. - async def get_resources_from_api(self): - return self.facade.database_blob_auditing_policies.get( + async def fetch_all(self): + policies = self.facade.database_blob_auditing_policies.get( self.resource_group_name, self.server_name, self.database_name) + self._parse_policies(policies) - def parse_resource(self, policies): + def _parse_policies(self, policies): self.update({ 'auditing_enabled': policies.state == "Enabled" }) From 74bb02f70bafa55d12784a2722036ff26601975c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 3 Mar 2019 12:37:28 -0500 Subject: [PATCH 172/667] Update database_threat_detection_policies.py --- .../sql/database_threat_detection_policies.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py b/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py index 84977378f..82a499f3b 100644 --- a/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py +++ b/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- -from ..resources import AzureSimpleResources +from ScoutSuite.providers.base.configs.resources import Resources -class DatabaseThreatDetectionPolicies(AzureSimpleResources): +class DatabaseThreatDetectionPolicies(Resources): def __init__(self, resource_group_name, server_name, database_name, facade): self.resource_group_name = resource_group_name @@ -12,11 +12,12 @@ def __init__(self, resource_group_name, server_name, database_name, facade): self.facade = facade # TODO: make it really async. - async def get_resources_from_api(self): - return self.facade.database_threat_detection_policies.get( + async def fetch_all(self): + policies = self.facade.database_threat_detection_policies.get( self.resource_group_name, self.server_name, self.database_name) + self._parse_policies(policies) - def parse_resource(self, policies): + def _parse_policies(self, policies): self.update({ 'threat_detection_enabled': policies.state == "Enabled" }) From ce0ffa60f9a3d4c57391f39251411704b2860fd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 3 Mar 2019 12:37:46 -0500 Subject: [PATCH 173/667] Update databases.py --- ScoutSuite/providers/azure/resources/sql/databases.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/databases.py b/ScoutSuite/providers/azure/resources/sql/databases.py index 4de894a47..0bd01e2a5 100644 --- a/ScoutSuite/providers/azure/resources/sql/databases.py +++ b/ScoutSuite/providers/azure/resources/sql/databases.py @@ -9,7 +9,7 @@ class Databases(AzureCompositeResources): - children = [ + _children = [ DatabaseBlobAuditingPolicies, DatabaseThreatDetectionPolicies, ReplicationLinks, @@ -33,7 +33,7 @@ async def fetch_all(self): self['databases'][db.name] = { 'id': db.name, } - await self.fetch_children( + await self._fetch_children( parent=self['databases'][db.name], resource_group_name=self.resource_group_name, server_name=self.server_name, From cfb3055c3abec8223960e4c5a0eac3c62c713b18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 3 Mar 2019 12:38:04 -0500 Subject: [PATCH 174/667] Update replication_links.py --- .../azure/resources/sql/replication_links.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/replication_links.py b/ScoutSuite/providers/azure/resources/sql/replication_links.py index f15e7e27c..bdd8323a9 100644 --- a/ScoutSuite/providers/azure/resources/sql/replication_links.py +++ b/ScoutSuite/providers/azure/resources/sql/replication_links.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- -from ..resources import AzureSimpleResources +from ScoutSuite.providers.base.configs.resources import Resources -class ReplicationLinks(AzureSimpleResources): +class ReplicationLinks(Resources): def __init__(self, resource_group_name, server_name, database_name, facade): self.resource_group_name = resource_group_name @@ -12,11 +12,12 @@ def __init__(self, resource_group_name, server_name, database_name, facade): self.facade = facade # TODO: make it really async. - async def get_resources_from_api(self): - return list(self.facade.replication_links.list_by_database( + async def fetch_all(self): + links = list(self.facade.replication_links.list_by_database( self.resource_group_name, self.server_name, self.database_name)) + self._parse_links(links) - def parse_resource(self, links): + def _parse_links(self, links): self.update({ 'replication_configured': len(links) > 0 }) From ddd0c08b14c49f2de76debb18f7058e5e92515a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 3 Mar 2019 12:38:18 -0500 Subject: [PATCH 175/667] Update servers.py --- ScoutSuite/providers/azure/resources/sql/servers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/servers.py b/ScoutSuite/providers/azure/resources/sql/servers.py index 9c268f95e..38fd1b256 100644 --- a/ScoutSuite/providers/azure/resources/sql/servers.py +++ b/ScoutSuite/providers/azure/resources/sql/servers.py @@ -11,7 +11,7 @@ class Servers(AzureCompositeResources): - children = [ + _children = [ Databases, ServerAzureAdAdministrators, ] @@ -30,7 +30,7 @@ async def fetch_all(self, credentials, **kwargs): 'id': id, 'name': server.name } - await self.fetch_children( + await self._fetch_children( parent=self['servers'][id], resource_group_name=resource_group_name, server_name=server.name, From 3f12a872a82238239395f3f0f06726d687aa8268 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sghaier?= Date: Sun, 3 Mar 2019 12:38:39 -0500 Subject: [PATCH 176/667] Update transparent_data_encryptions.py --- .../resources/sql/transparent_data_encryptions.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py b/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py index d82b51fce..96e64efa1 100644 --- a/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py +++ b/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- -from ..resources import AzureSimpleResources +from ScoutSuite.providers.base.configs.resources import Resources -class TransparentDataEncryptions(AzureSimpleResources): +class TransparentDataEncryptions(Resources): def __init__(self, resource_group_name, server_name, database_name, facade): self.resource_group_name = resource_group_name @@ -12,11 +12,12 @@ def __init__(self, resource_group_name, server_name, database_name, facade): self.facade = facade # TODO: make it really async. - async def get_resources_from_api(self): - return self.facade.transparent_data_encryptions.get( + async def fetch_all(self): + encryptions = self.facade.transparent_data_encryptions.get( self.resource_group_name, self.server_name, self.database_name) + self._parse_encryptions(encryptions) - def parse_resource(self, encryptions): - return self.update({ + def _parse_encryptions(self, encryptions): + self.update({ 'transparent_data_encryption_enabled': encryptions.status == "Enabled" }) From b05d24ff9f681ca8a9d6a01e87fe7ec6d8162e76 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Sun, 3 Mar 2019 20:19:14 -0500 Subject: [PATCH 177/667] Moved migrated services to resources --- ScoutSuite/providers/aws/configs/services.py | 4 ++-- .../aws/{services => resources}/awslambda/service.py | 0 ScoutSuite/providers/aws/{services => resources}/ec2/ami.py | 0 .../providers/aws/{services => resources}/ec2/instances.py | 0 .../aws/{services => resources}/ec2/networkinterfaces.py | 0 .../aws/{services => resources}/ec2/securitygroups.py | 0 .../providers/aws/{services => resources}/ec2/service.py | 0 .../providers/aws/{services => resources}/ec2/snapshots.py | 0 .../providers/aws/{services => resources}/ec2/volumes.py | 0 ScoutSuite/providers/aws/{services => resources}/ec2/vpcs.py | 0 10 files changed, 2 insertions(+), 2 deletions(-) rename ScoutSuite/providers/aws/{services => resources}/awslambda/service.py (100%) rename ScoutSuite/providers/aws/{services => resources}/ec2/ami.py (100%) rename ScoutSuite/providers/aws/{services => resources}/ec2/instances.py (100%) rename ScoutSuite/providers/aws/{services => resources}/ec2/networkinterfaces.py (100%) rename ScoutSuite/providers/aws/{services => resources}/ec2/securitygroups.py (100%) rename ScoutSuite/providers/aws/{services => resources}/ec2/service.py (100%) rename ScoutSuite/providers/aws/{services => resources}/ec2/snapshots.py (100%) rename ScoutSuite/providers/aws/{services => resources}/ec2/volumes.py (100%) rename ScoutSuite/providers/aws/{services => resources}/ec2/vpcs.py (100%) diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index a6b340ad7..380666fa2 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -1,11 +1,11 @@ # -*- coding: utf-8 -*- -from ScoutSuite.providers.aws.services.awslambda.service import Lambdas +from ScoutSuite.providers.aws.resources.awslambda.service import Lambdas from ScoutSuite.providers.aws.services.cloudformation import CloudFormationConfig from ScoutSuite.providers.aws.services.cloudtrail import CloudTrailConfig from ScoutSuite.providers.aws.services.cloudwatch import CloudWatchConfig from ScoutSuite.providers.aws.services.directconnect import DirectConnectConfig -from ScoutSuite.providers.aws.services.ec2.service import EC2 +from ScoutSuite.providers.aws.resources.ec2.service import EC2 from ScoutSuite.providers.aws.services.efs import EFSConfig from ScoutSuite.providers.aws.services.elasticache import ElastiCacheConfig from ScoutSuite.providers.aws.services.elb import ELBConfig diff --git a/ScoutSuite/providers/aws/services/awslambda/service.py b/ScoutSuite/providers/aws/resources/awslambda/service.py similarity index 100% rename from ScoutSuite/providers/aws/services/awslambda/service.py rename to ScoutSuite/providers/aws/resources/awslambda/service.py diff --git a/ScoutSuite/providers/aws/services/ec2/ami.py b/ScoutSuite/providers/aws/resources/ec2/ami.py similarity index 100% rename from ScoutSuite/providers/aws/services/ec2/ami.py rename to ScoutSuite/providers/aws/resources/ec2/ami.py diff --git a/ScoutSuite/providers/aws/services/ec2/instances.py b/ScoutSuite/providers/aws/resources/ec2/instances.py similarity index 100% rename from ScoutSuite/providers/aws/services/ec2/instances.py rename to ScoutSuite/providers/aws/resources/ec2/instances.py diff --git a/ScoutSuite/providers/aws/services/ec2/networkinterfaces.py b/ScoutSuite/providers/aws/resources/ec2/networkinterfaces.py similarity index 100% rename from ScoutSuite/providers/aws/services/ec2/networkinterfaces.py rename to ScoutSuite/providers/aws/resources/ec2/networkinterfaces.py diff --git a/ScoutSuite/providers/aws/services/ec2/securitygroups.py b/ScoutSuite/providers/aws/resources/ec2/securitygroups.py similarity index 100% rename from ScoutSuite/providers/aws/services/ec2/securitygroups.py rename to ScoutSuite/providers/aws/resources/ec2/securitygroups.py diff --git a/ScoutSuite/providers/aws/services/ec2/service.py b/ScoutSuite/providers/aws/resources/ec2/service.py similarity index 100% rename from ScoutSuite/providers/aws/services/ec2/service.py rename to ScoutSuite/providers/aws/resources/ec2/service.py diff --git a/ScoutSuite/providers/aws/services/ec2/snapshots.py b/ScoutSuite/providers/aws/resources/ec2/snapshots.py similarity index 100% rename from ScoutSuite/providers/aws/services/ec2/snapshots.py rename to ScoutSuite/providers/aws/resources/ec2/snapshots.py diff --git a/ScoutSuite/providers/aws/services/ec2/volumes.py b/ScoutSuite/providers/aws/resources/ec2/volumes.py similarity index 100% rename from ScoutSuite/providers/aws/services/ec2/volumes.py rename to ScoutSuite/providers/aws/resources/ec2/volumes.py diff --git a/ScoutSuite/providers/aws/services/ec2/vpcs.py b/ScoutSuite/providers/aws/resources/ec2/vpcs.py similarity index 100% rename from ScoutSuite/providers/aws/services/ec2/vpcs.py rename to ScoutSuite/providers/aws/resources/ec2/vpcs.py From 63f772ec7cd4c1b5dcc476fd660deb198860a6b8 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Sun, 3 Mar 2019 20:26:04 -0500 Subject: [PATCH 178/667] Fixed bad imports --- ScoutSuite/providers/aws/resources/ec2/service.py | 8 ++++---- ScoutSuite/providers/aws/resources/ec2/vpcs.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/ec2/service.py b/ScoutSuite/providers/aws/resources/ec2/service.py index 404b8906c..736a76b81 100644 --- a/ScoutSuite/providers/aws/resources/ec2/service.py +++ b/ScoutSuite/providers/aws/resources/ec2/service.py @@ -1,8 +1,8 @@ from ScoutSuite.providers.aws.resources.regions import Regions -from ScoutSuite.providers.aws.services.ec2.ami import AmazonMachineImages -from ScoutSuite.providers.aws.services.ec2.vpcs import Vpcs -from ScoutSuite.providers.aws.services.ec2.snapshots import Snapshots -from ScoutSuite.providers.aws.services.ec2.volumes import Volumes +from ScoutSuite.providers.aws.resources.ec2.ami import AmazonMachineImages +from ScoutSuite.providers.aws.resources.ec2.vpcs import Vpcs +from ScoutSuite.providers.aws.resources.ec2.snapshots import Snapshots +from ScoutSuite.providers.aws.resources.ec2.volumes import Volumes class EC2(Regions): diff --git a/ScoutSuite/providers/aws/resources/ec2/vpcs.py b/ScoutSuite/providers/aws/resources/ec2/vpcs.py index 5ddba3fd4..7b5c6ceeb 100644 --- a/ScoutSuite/providers/aws/resources/ec2/vpcs.py +++ b/ScoutSuite/providers/aws/resources/ec2/vpcs.py @@ -1,8 +1,8 @@ from ScoutSuite.providers.aws.resources.resources import AWSCompositeResources from ScoutSuite.providers.aws.facade.facade import AWSFacade -from ScoutSuite.providers.aws.services.ec2.instances import EC2Instances -from ScoutSuite.providers.aws.services.ec2.securitygroups import SecurityGroups -from ScoutSuite.providers.aws.services.ec2.networkinterfaces import NetworkInterfaces +from ScoutSuite.providers.aws.resources.ec2.instances import EC2Instances +from ScoutSuite.providers.aws.resources.ec2.securitygroups import SecurityGroups +from ScoutSuite.providers.aws.resources.ec2.networkinterfaces import NetworkInterfaces class Vpcs(AWSCompositeResources): From 2a2f22fd32307d9f6f1ed6e4fc9ba9dcf7ba89c1 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Sun, 3 Mar 2019 21:43:20 -0500 Subject: [PATCH 179/667] Implemented cloudformation facade --- .../providers/aws/facade/cloudformation.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 ScoutSuite/providers/aws/facade/cloudformation.py diff --git a/ScoutSuite/providers/aws/facade/cloudformation.py b/ScoutSuite/providers/aws/facade/cloudformation.py new file mode 100644 index 000000000..655433b90 --- /dev/null +++ b/ScoutSuite/providers/aws/facade/cloudformation.py @@ -0,0 +1,20 @@ +import json + +from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils + + +class CloudFormation: + def get_stacks(self, region: str): + stacks = AWSFacadeUtils.get_all_pages('cloudformation', region, 'list_stacks', 'StackSummaries') + + client = AWSFacadeUtils.get_client('cloudformation', region) + for stack in stacks: + stack_name = stack['StackName'] + stack_description = client.describe_stacks(StackName=stack_name)['Stacks'][0] + stack.update(stack_description) + stack['template'] = client.get_template(StackName=stack_name)['TemplateBody'] + stack_policy = client.get_stack_policy(StackName=stack_name) + if 'StackPolicyBody' in stack_policy: + stack['policy'] = json.loads(stack_policy['StackPolicyBody']) + + return stacks From f312446bd2373e9bdfb0ed7d38a0df214f74b7c8 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Sun, 3 Mar 2019 21:43:35 -0500 Subject: [PATCH 180/667] Integrated cloudformation facade to aws facade --- ScoutSuite/providers/aws/facade/facade.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ScoutSuite/providers/aws/facade/facade.py b/ScoutSuite/providers/aws/facade/facade.py index baf02896d..7d22ae458 100644 --- a/ScoutSuite/providers/aws/facade/facade.py +++ b/ScoutSuite/providers/aws/facade/facade.py @@ -2,11 +2,13 @@ from collections import Counter from ScoutSuite.providers.aws.facade.ec2 import EC2Facade from ScoutSuite.providers.aws.facade.awslambda import LambdaFacade +from ScoutSuite.providers.aws.facade.cloudformation import CloudFormation class AWSFacade(object): def __init__(self): self.ec2 = EC2Facade() self.awslambda = LambdaFacade() + self.cloudformation = CloudFormation() async def build_region_list(self, service: str, chosen_regions=None, partition_name='aws'): From 367e9e35f7e6473e17c10f83753a977b2856798f Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Sun, 3 Mar 2019 21:43:43 -0500 Subject: [PATCH 181/667] Removed unused code --- ScoutSuite/providers/base/configs/services.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ScoutSuite/providers/base/configs/services.py b/ScoutSuite/providers/base/configs/services.py index ccbc2992a..3665c6dbd 100644 --- a/ScoutSuite/providers/base/configs/services.py +++ b/ScoutSuite/providers/base/configs/services.py @@ -38,7 +38,3 @@ async def fetch(self, credentials, services=None, regions=None): except Exception as e: print_error('Error: could not fetch %s configuration.' % service) print_exception(e) - - # TODO is this ever called? - # def postprocessing(self): - # pass From 0ada1d8ccca9e8448026522d2e5432a00b396884 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Sun, 3 Mar 2019 21:45:46 -0500 Subject: [PATCH 182/667] Migrated cloudformation service to the new architecture --- .../aws/resources/cloudformation/service.py | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 ScoutSuite/providers/aws/resources/cloudformation/service.py diff --git a/ScoutSuite/providers/aws/resources/cloudformation/service.py b/ScoutSuite/providers/aws/resources/cloudformation/service.py new file mode 100644 index 000000000..70ce43ea5 --- /dev/null +++ b/ScoutSuite/providers/aws/resources/cloudformation/service.py @@ -0,0 +1,35 @@ +from ScoutSuite.providers.aws.resources.regions import Regions +from ScoutSuite.providers.aws.resources.resources import AWSResources +from ScoutSuite.providers.aws.facade.facade import AWSFacade + + +class Stack(AWSResources): + async def fetch_all(self, **kwargs): + raw_stacks = self.facade.cloudformation.get_stacks(self.scope['region']) + for raw_stack in raw_stacks: + name, stack = self._parse_stack(raw_stack) + self[name] = stack + + def _parse_stack(self, raw_stack): + raw_stack['id'] = raw_stack.pop('StackId') + raw_stack['name'] = raw_stack.pop('StackName') + raw_stack['drifted'] = raw_stack.pop('DriftInformation')['StackDriftStatus'] == 'DRIFTED' + + template = raw_stack.pop('template') + raw_stack['deletion_policy'] = 'Delete' + for group in template.keys(): + if 'DeletionPolicy' in template[group]: + raw_stack['deletion_policy'] = template[group]['DeletionPolicy'] + break + + return raw_stack['name'], raw_stack + + + +class CloudFormation(Regions): + _children = [ + (Stack, 'functions') + ] + + def __init__(self): + super(CloudFormation, self).__init__('cloudformation') From f32aeafedbd2b21b36c1fcac077087bdbaeceeb2 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Sun, 3 Mar 2019 21:46:06 -0500 Subject: [PATCH 183/667] Integrated new cloudformation implementation --- .../aws/services.cloudformation.regions.id.stacks.html | 2 +- ScoutSuite/providers/aws/configs/services.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ScoutSuite/output/data/html/partials/aws/services.cloudformation.regions.id.stacks.html b/ScoutSuite/output/data/html/partials/aws/services.cloudformation.regions.id.stacks.html index 93d540637..258c5ca3f 100644 --- a/ScoutSuite/output/data/html/partials/aws/services.cloudformation.regions.id.stacks.html +++ b/ScoutSuite/output/data/html/partials/aws/services.cloudformation.regions.id.stacks.html @@ -19,7 +19,7 @@

    Information

    None {{/if}}
    -
    Termination protection enabled: {{termination_protection}}
    +
    Termination protection enabled: {{EnableTerminationProtection}}
    Configuration has drifted: {{drifted}}
    Deletion policy: {{deletion_policy}}
    diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index 380666fa2..b02084149 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from ScoutSuite.providers.aws.resources.awslambda.service import Lambdas -from ScoutSuite.providers.aws.services.cloudformation import CloudFormationConfig +from ScoutSuite.providers.aws.resources.cloudformation.service import CloudFormation from ScoutSuite.providers.aws.services.cloudtrail import CloudTrailConfig from ScoutSuite.providers.aws.services.cloudwatch import CloudWatchConfig from ScoutSuite.providers.aws.services.directconnect import DirectConnectConfig @@ -54,7 +54,7 @@ class AWSServicesConfig(BaseServicesConfig): def __init__(self, metadata=None, thread_config=4, **kwargs): super().__init__(metadata, thread_config) - self.cloudformation = CloudFormationConfig(metadata['management']['cloudformation'], thread_config) + self.cloudformation = CloudFormation() self.cloudtrail = CloudTrailConfig(metadata['management']['cloudtrail'], thread_config) self.cloudwatch = CloudWatchConfig(metadata['management']['cloudwatch'], thread_config) self.directconnect = DirectConnectConfig(metadata['network']['directconnect'], thread_config) From 872b90701473232bab233c69c9bd651b8d76a699 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Sun, 3 Mar 2019 21:46:19 -0500 Subject: [PATCH 184/667] Removed unused imports --- ScoutSuite/providers/aws/facade/awslambda.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/awslambda.py b/ScoutSuite/providers/aws/facade/awslambda.py index adb8cfeca..04c3649d3 100644 --- a/ScoutSuite/providers/aws/facade/awslambda.py +++ b/ScoutSuite/providers/aws/facade/awslambda.py @@ -1,7 +1,3 @@ -import boto3 -import base64 -import itertools - from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils From 1dda2d12f5be5cc20aafcbea8a3415c6eb021a5c Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Sun, 3 Mar 2019 21:53:32 -0500 Subject: [PATCH 185/667] Fixed ec2 processing bug --- ScoutSuite/providers/aws/provider.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index 74cf3cea5..12f0d6148 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -67,13 +67,15 @@ def preprocessing(self, ip_ranges=None, ip_ranges_name_key=None): # self.parse_elb_policies() # Various data processing calls - self._check_ec2_zone_distribution() - self._add_security_group_name_to_ec2_grants() - self._add_last_snapshot_date_to_ec2_volumes() + if 'ec2' in self.service_list: + self._check_ec2_zone_distribution() + self._add_security_group_name_to_ec2_grants() + self._add_last_snapshot_date_to_ec2_volumes() + self._match_instances_and_roles() + self._process_cloudtrail_trails(self.services['cloudtrail']) self._add_cidr_display_name(ip_ranges, ip_ranges_name_key) self._merge_route53_and_route53domains() - self._match_instances_and_roles() self._match_iam_policies_and_buckets() super(AWSProvider, self).preprocessing() From e8dc007752b06c99ec2e59246c3c5f8d29c6ec1c Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Sun, 3 Mar 2019 21:56:42 -0500 Subject: [PATCH 186/667] Fixed wrong service key --- ScoutSuite/providers/aws/resources/cloudformation/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/resources/cloudformation/service.py b/ScoutSuite/providers/aws/resources/cloudformation/service.py index 70ce43ea5..150fa4a8c 100644 --- a/ScoutSuite/providers/aws/resources/cloudformation/service.py +++ b/ScoutSuite/providers/aws/resources/cloudformation/service.py @@ -28,7 +28,7 @@ def _parse_stack(self, raw_stack): class CloudFormation(Regions): _children = [ - (Stack, 'functions') + (Stack, 'stacks') ] def __init__(self): From 142a126ad4a28d9c203c2b9da09ad7f4f1429fc8 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Sun, 3 Mar 2019 22:06:02 -0500 Subject: [PATCH 187/667] Removed old implementation --- .../providers/aws/services/cloudformation.py | 58 ------------------- 1 file changed, 58 deletions(-) delete mode 100644 ScoutSuite/providers/aws/services/cloudformation.py diff --git a/ScoutSuite/providers/aws/services/cloudformation.py b/ScoutSuite/providers/aws/services/cloudformation.py deleted file mode 100644 index bef5b20f5..000000000 --- a/ScoutSuite/providers/aws/services/cloudformation.py +++ /dev/null @@ -1,58 +0,0 @@ -# -*- coding: utf-8 -*- - -import json - -from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients - - -######################################## -# CloudFormationRegionConfig -######################################## - -class CloudFormationRegionConfig(RegionConfig): - """ - CloudFormation configuration for a single AWS region - """ - stacks = {} - - def parse_stack(self, global_params, region, stack): - """ - Parse a single stack and fetch additional attributes - - :param stack: - :param global_params: Parameters shared for all regions - :param region: Name of the AWS region - """ - stack['id'] = stack.pop('StackId') - stack['name'] = stack.pop('StackName') - - stack_description = api_clients[region].describe_stacks(StackName=stack['name']) - stack['termination_protection'] = stack_description['Stacks'][0]['EnableTerminationProtection'] - stack['drifted'] = stack.pop('DriftInformation')['StackDriftStatus'] == 'DRIFTED' - - template = api_clients[region].get_template(StackName=stack['name'])['TemplateBody']['Resources'] - stack['deletion_policy'] = 'Delete' - for group in template.keys(): - if 'DeletionPolicy' in template[group]: - stack['deletion_policy'] = template[group]['DeletionPolicy'] - break - - stack_policy = api_clients[region].get_stack_policy(StackName=stack['name']) - if 'StackPolicyBody' in stack_policy: - stack['policy'] = json.loads(stack_policy['StackPolicyBody']) - self.stacks[stack['name']] = stack - - -######################################## -# CloudFormationConfig -######################################## - -class CloudFormationConfig(RegionalServiceConfig): - """ - CloudFormation configuration for all AWS regions - """ - - region_config_class = CloudFormationRegionConfig - - def __init__(self, service_metadata, thread_config=4): - super(CloudFormationConfig, self).__init__(service_metadata, thread_config) From 8645af9551bb5b88c0857e035dddc7323223818f Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 4 Mar 2019 10:12:16 -0500 Subject: [PATCH 188/667] Update ScoutSuite/providers/azure/resources/utils.py Co-Authored-By: misg --- ScoutSuite/providers/azure/resources/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/azure/resources/utils.py b/ScoutSuite/providers/azure/resources/utils.py index 3011743f7..d1340ae97 100644 --- a/ScoutSuite/providers/azure/resources/utils.py +++ b/ScoutSuite/providers/azure/resources/utils.py @@ -3,7 +3,7 @@ def get_non_provider_id(name): """ - Not all resources have an ID and some services allow the use of "." in names, which break's Scout2's + Not all resources have an ID and some services allow the use of "." in names, which breaks Scout's recursion scheme if name is used as an ID. Use SHA1(name) instead. :param name: Name of the resource to From 0737810fbc909ea5ead500940b3f955202702f25 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 4 Mar 2019 10:59:54 -0500 Subject: [PATCH 189/667] Implemented CloudTrail facade --- ScoutSuite/providers/aws/facade/cloudtrail.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 ScoutSuite/providers/aws/facade/cloudtrail.py diff --git a/ScoutSuite/providers/aws/facade/cloudtrail.py b/ScoutSuite/providers/aws/facade/cloudtrail.py new file mode 100644 index 000000000..8836e7f63 --- /dev/null +++ b/ScoutSuite/providers/aws/facade/cloudtrail.py @@ -0,0 +1,13 @@ +from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils + + +class CloudTrailFacade: + def get_trails(self, region): + client = AWSFacadeUtils.get_client('cloudtrail', region) + trails = client.describe_trails()['trailList'] + + for trail in trails: + trail.update(client.get_trail_status(Name=trail['TrailARN'])) + trail['EventSelectors'] = client.get_event_selectors(TrailName=trail['TrailARN'])['EventSelectors'] + + return trails \ No newline at end of file From b4e22f937b2a7f4aead7f6dd9532def4094e999b Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 4 Mar 2019 11:00:04 -0500 Subject: [PATCH 190/667] Added CloudTrail facade to AWS facade --- ScoutSuite/providers/aws/facade/facade.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ScoutSuite/providers/aws/facade/facade.py b/ScoutSuite/providers/aws/facade/facade.py index baf02896d..ae29dd9bc 100644 --- a/ScoutSuite/providers/aws/facade/facade.py +++ b/ScoutSuite/providers/aws/facade/facade.py @@ -2,11 +2,13 @@ from collections import Counter from ScoutSuite.providers.aws.facade.ec2 import EC2Facade from ScoutSuite.providers.aws.facade.awslambda import LambdaFacade +from ScoutSuite.providers.aws.facade.cloudtrail import CloudTrailFacade class AWSFacade(object): def __init__(self): self.ec2 = EC2Facade() self.awslambda = LambdaFacade() + self.cloudtrail = CloudTrailFacade() async def build_region_list(self, service: str, chosen_regions=None, partition_name='aws'): From 0d90c881dff0404083ca81678ae895ef74d15be2 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 4 Mar 2019 11:00:30 -0500 Subject: [PATCH 191/667] Added a check for CloudTrail preprocessing --- ScoutSuite/providers/aws/provider.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index 12f0d6148..c3974bee9 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -73,7 +73,9 @@ def preprocessing(self, ip_ranges=None, ip_ranges_name_key=None): self._add_last_snapshot_date_to_ec2_volumes() self._match_instances_and_roles() - self._process_cloudtrail_trails(self.services['cloudtrail']) + if 'cloudtrail' in self.service_list: + self._process_cloudtrail_trails(self.services['cloudtrail']) + self._add_cidr_display_name(ip_ranges, ip_ranges_name_key) self._merge_route53_and_route53domains() self._match_iam_policies_and_buckets() From 213e047193f92d013083616a3ae1947c616d14d3 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 4 Mar 2019 11:00:48 -0500 Subject: [PATCH 192/667] Moved get_non_provider_id to base utils --- ScoutSuite/providers/base/utils.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 ScoutSuite/providers/base/utils.py diff --git a/ScoutSuite/providers/base/utils.py b/ScoutSuite/providers/base/utils.py new file mode 100644 index 000000000..a944d0ae6 --- /dev/null +++ b/ScoutSuite/providers/base/utils.py @@ -0,0 +1,14 @@ +from hashlib import sha1 + + +def get_non_provider_id(name): + """ + Not all AWS resources have an ID and some services allow the use of "." in names, which break's Scout2's + recursion scheme if name is used as an ID. Use SHA1(name) instead. + + :param name: Name of the resource to + :return: SHA1(name) + """ + m = sha1() + m.update(name.encode('utf-8')) + return m.hexdigest() From c5b5b07ccdbcbc0530237a4202a6254106f21d3b Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 4 Mar 2019 11:01:09 -0500 Subject: [PATCH 193/667] Migrated CloudTrail to the new architecture --- .../aws/resources/cloudtrail/service.py | 177 ++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 ScoutSuite/providers/aws/resources/cloudtrail/service.py diff --git a/ScoutSuite/providers/aws/resources/cloudtrail/service.py b/ScoutSuite/providers/aws/resources/cloudtrail/service.py new file mode 100644 index 000000000..381b57de6 --- /dev/null +++ b/ScoutSuite/providers/aws/resources/cloudtrail/service.py @@ -0,0 +1,177 @@ +from ScoutSuite.providers.aws.resources.regions import Regions +from ScoutSuite.providers.aws.resources.resources import AWSResources +from ScoutSuite.providers.aws.facade.facade import AWSFacade +from ScoutSuite.providers.base.utils import get_non_provider_id + +import time + +class Trails(AWSResources): + async def fetch_all(self, **kwargs): + raw_trails = self.facade.cloudtrail.get_trails(self.scope['region']) + for raw_trail in raw_trails: + name, resource = self._parse_trail(raw_trail) + self[name] = resource + + def _parse_trail(self, raw_trail): + trail = {'name': raw_trail.pop('Name')} + trail_id = get_non_provider_id(trail['name']) + + # Do not duplicate entries for multiregion trails + if 'IsMultiRegionTrail' in raw_trail and raw_trail['IsMultiRegionTrail'] and raw_trail['HomeRegion'] != self.scope['region']: + for key in ['HomeRegion', 'TrailARN']: + trail[key] = raw_trail[key] + trail['scout2_link'] = 'services.cloudtrail.regions.%s.trails.%s' % (raw_trail['HomeRegion'], trail_id) + return trail_id, trail + + for key in raw_trail: + trail[key] = raw_trail[key] + trail['bucket_id'] = get_non_provider_id(trail.pop('S3BucketName')) + for key in ['IsMultiRegionTrail', 'LogFileValidationEnabled']: + if key not in trail: + trail[key] = False + + for key in ['KMSKeyId', 'IsLogging', 'LatestDeliveryTime', 'LatestDeliveryError', 'StartLoggingTime', + 'StopLoggingTime', 'LatestNotificationTime', 'LatestNotificationError', + 'LatestCloudWatchLogsDeliveryError', 'LatestCloudWatchLogsDeliveryTime']: + trail[key] = trail[key] if key in trail else None + + # using trail ARN instead of name as with Organizations the trail would be located in another account + trail['wildcard_data_logging'] = self.data_logging_status(trail) + + for event_selector in trail['EventSelectors']: + trail['DataEventsEnabled'] = len(event_selector['DataResources']) > 0 + trail['ManagementEventsEnabled'] = event_selector['IncludeManagementEvents'] + + return trail_id, trail + + def data_logging_status(self, trail): + for event_selector in trail['EventSelectors']: + has_wildcard = {u'Values': [u'arn:aws:s3:::'], u'Type': u'AWS::S3::Object'} in event_selector['DataResources'] + is_logging = trail['IsLogging'] + + if has_wildcard and is_logging and self.is_fresh(trail): + return True + + return False + + @staticmethod + def is_fresh(trail_details): + delivery_time = trail_details.get('LatestCloudWatchLogsDeliveryTime', "9999999").strftime("%s") + delivery_age = ((int(time.time()) - int(delivery_time)) / 1440) + return delivery_age <= 24 + + +class CloudTrail(Regions): + _children = [ + (Trails, 'trails') + ] + + def __init__(self): + super(CloudTrail, self).__init__('cloudtrail') + + +# # -*- coding: utf-8 -*- +# """ +# CloudTrail-related classes and functions +# """ +# import time + +# from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients + + +# ######################################## +# # CloudTrailRegionConfig +# ######################################## + +# class CloudTrailRegionConfig(RegionConfig): +# """ +# CloudTrail configuration for a single AWS region +# """ +# trails = {} + +# def parse_trail(self, global_params, region, trail): +# """ +# Parse a single CloudTrail trail + +# :param trail: +# :param global_params: Parameters shared for all regions +# :param region: Name of the AWS region +# """ +# trail_config = {'name': trail.pop('Name')} +# trail_id = self.get_non_provider_id(trail_config['name']) +# trail_details = None + +# api_client = api_clients[region] + +# # Do not duplicate entries for multiregion trails +# if 'IsMultiRegionTrail' in trail and trail['IsMultiRegionTrail'] and trail['HomeRegion'] != region: +# for key in ['HomeRegion', 'TrailARN']: +# trail_config[key] = trail[key] +# trail_config['scout2_link'] = 'services.cloudtrail.regions.%s.trails.%s' % (trail['HomeRegion'], trail_id) +# else: +# for key in trail: +# trail_config[key] = trail[key] +# trail_config['bucket_id'] = self.get_non_provider_id(trail_config.pop('S3BucketName')) +# for key in ['IsMultiRegionTrail', 'LogFileValidationEnabled']: +# if key not in trail_config: +# trail_config[key] = False +# trail_details = api_client.get_trail_status(Name=trail['TrailARN']) +# for key in ['KMSKeyId', 'IsLogging', 'LatestDeliveryTime', 'LatestDeliveryError', 'StartLoggingTime', +# 'StopLoggingTime', 'LatestNotificationTime', 'LatestNotificationError', +# 'LatestCloudWatchLogsDeliveryError', 'LatestCloudWatchLogsDeliveryTime']: +# trail_config[key] = trail_details[key] if key in trail_details else None + +# if trail_details: +# # using trail ARN instead of name as with Organizations the trail would be located in another account +# trail_config['wildcard_data_logging'] = self.data_logging_status(trail_config['TrailARN'], +# trail_details, +# api_client) + +# for es in api_client.get_event_selectors(TrailName=trail_config['TrailARN'])['EventSelectors']: +# trail_config['DataEventsEnabled'] = len(es['DataResources']) > 0 +# trail_config['ManagementEventsEnabled'] = es['IncludeManagementEvents'] + +# self.trails[trail_id] = trail_config + + + + +# ######################################## +# # CloudTrailConfig +# ######################################## + +# class CloudTrailConfig(RegionalServiceConfig): +# """ +# CloudTrail configuration for all AWS regions +# """ + +# region_config_class = CloudTrailRegionConfig + +# def __init__(self, service_metadata, thread_config=4): +# super(CloudTrailConfig, self).__init__(service_metadata, thread_config) + + +# ######################################## +# # Post Processing +# ######################################## + +# def cloudtrail_postprocessing(aws_config): +# cloudtrail_config = aws_config['services']['cloudtrail'] +# # Global services logging duplicated +# if 'cloudtrail-duplicated-global-services-logging' in cloudtrail_config['violations']: +# if len(cloudtrail_config['violations']['cloudtrail-duplicated-global-services-logging']['items']) < 2: +# cloudtrail_config['violations']['cloudtrail-duplicated-global-services-logging']['items'] = [] +# cloudtrail_config['violations']['cloudtrail-duplicated-global-services-logging']['flagged_items'] = 0 +# # Global services logging disabled +# if 'cloudtrail-no-global-services-logging' in cloudtrail_config['violations']: +# if len(cloudtrail_config['violations']['cloudtrail-no-global-services-logging']['items']) != \ +# cloudtrail_config['violations']['cloudtrail-no-global-services-logging']['checked_items']: +# cloudtrail_config['violations']['cloudtrail-no-global-services-logging']['items'] = [] +# cloudtrail_config['violations']['cloudtrail-no-global-services-logging']['flagged_items'] = 0 +# # CloudTrail not enabled at all... +# if not sum(cloudtrail_config['regions'][r]['trails_count'] for r in cloudtrail_config['regions']): +# for r in cloudtrail_config['regions']: +# cloudtrail_config['violations']['cloudtrail-no-logging']['items'].append('cloudtrail.regions.%s' % r) +# cloudtrail_config['violations']['cloudtrail-no-logging']['checked_items'] += 1 +# cloudtrail_config['violations']['cloudtrail-no-logging']['flagged_items'] += 1 +# cloudtrail_config['violations']['cloudtrail-no-logging']['dashboard_name'] = 'Regions' From 0e1fc2a95a2977d9e1e06ae71fe55c701e5d83c7 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 4 Mar 2019 11:01:50 -0500 Subject: [PATCH 194/667] Integrated new implementation of CloudTrail, deleted previous --- ScoutSuite/providers/aws/configs/services.py | 27 ++-- .../providers/aws/services/cloudtrail.py | 119 ------------------ 2 files changed, 17 insertions(+), 129 deletions(-) delete mode 100644 ScoutSuite/providers/aws/services/cloudtrail.py diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index 380666fa2..4234c172e 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -2,7 +2,7 @@ from ScoutSuite.providers.aws.resources.awslambda.service import Lambdas from ScoutSuite.providers.aws.services.cloudformation import CloudFormationConfig -from ScoutSuite.providers.aws.services.cloudtrail import CloudTrailConfig +from ScoutSuite.providers.aws.resources.cloudtrail.service import CloudTrail from ScoutSuite.providers.aws.services.cloudwatch import CloudWatchConfig from ScoutSuite.providers.aws.services.directconnect import DirectConnectConfig from ScoutSuite.providers.aws.resources.ec2.service import EC2 @@ -35,7 +35,7 @@ class AWSServicesConfig(BaseServicesConfig): """ Object that holds the necessary AWS configuration for all services in scope. - + :ivar cloudtrail: CloudTrail configuration :ivar cloudwatch: CloudWatch configuration: TODO :ivar config: Config configuration @@ -54,19 +54,24 @@ class AWSServicesConfig(BaseServicesConfig): def __init__(self, metadata=None, thread_config=4, **kwargs): super().__init__(metadata, thread_config) - self.cloudformation = CloudFormationConfig(metadata['management']['cloudformation'], thread_config) - self.cloudtrail = CloudTrailConfig(metadata['management']['cloudtrail'], thread_config) - self.cloudwatch = CloudWatchConfig(metadata['management']['cloudwatch'], thread_config) - self.directconnect = DirectConnectConfig(metadata['network']['directconnect'], thread_config) + self.cloudformation = CloudFormationConfig( + metadata['management']['cloudformation'], thread_config) + self.cloudtrail = CloudTrail() + self.cloudwatch = CloudWatchConfig( + metadata['management']['cloudwatch'], thread_config) + self.directconnect = DirectConnectConfig( + metadata['network']['directconnect'], thread_config) self.ec2 = EC2() self.efs = EFSConfig(metadata['storage']['efs'], thread_config) - self.elasticache = ElastiCacheConfig(metadata['database']['elasticache'], thread_config) + self.elasticache = ElastiCacheConfig( + metadata['database']['elasticache'], thread_config) self.elb = ELBConfig(metadata['compute']['elb'], thread_config) self.elbv2 = ELBv2Config(metadata['compute']['elbv2'], thread_config) self.emr = EMRConfig(metadata['analytics']['emr'], thread_config) self.iam = IAMConfig(thread_config) self.awslambda = Lambdas() - self.redshift = RedshiftConfig(metadata['database']['redshift'], thread_config) + self.redshift = RedshiftConfig( + metadata['database']['redshift'], thread_config) self.rds = RDSConfig(metadata['database']['rds'], thread_config) self.route53 = Route53Config(thread_config) self.route53domains = Route53DomainsConfig(thread_config) @@ -77,8 +82,10 @@ def __init__(self, metadata=None, thread_config=4, **kwargs): self.vpc = VPCConfig(metadata['network']['vpc'], thread_config) try: - self.config = ConfigConfig(metadata['management']['config'], thread_config) - self.dynamodb = DynamoDBConfig(metadata['database']['dynamodb'], thread_config) + self.config = ConfigConfig( + metadata['management']['config'], thread_config) + self.dynamodb = DynamoDBConfig( + metadata['database']['dynamodb'], thread_config) self.kms = KMSConfig(metadata['security']['kms'], thread_config) except (NameError, TypeError): pass diff --git a/ScoutSuite/providers/aws/services/cloudtrail.py b/ScoutSuite/providers/aws/services/cloudtrail.py deleted file mode 100644 index 05cca58bf..000000000 --- a/ScoutSuite/providers/aws/services/cloudtrail.py +++ /dev/null @@ -1,119 +0,0 @@ -# -*- coding: utf-8 -*- -""" -CloudTrail-related classes and functions -""" -import time - -from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients - - -######################################## -# CloudTrailRegionConfig -######################################## - -class CloudTrailRegionConfig(RegionConfig): - """ - CloudTrail configuration for a single AWS region - """ - trails = {} - - def parse_trail(self, global_params, region, trail): - """ - Parse a single CloudTrail trail - - :param trail: - :param global_params: Parameters shared for all regions - :param region: Name of the AWS region - """ - trail_config = {'name': trail.pop('Name')} - trail_id = self.get_non_provider_id(trail_config['name']) - trail_details = None - - api_client = api_clients[region] - - # Do not duplicate entries for multiregion trails - if 'IsMultiRegionTrail' in trail and trail['IsMultiRegionTrail'] and trail['HomeRegion'] != region: - for key in ['HomeRegion', 'TrailARN']: - trail_config[key] = trail[key] - trail_config['scout2_link'] = 'services.cloudtrail.regions.%s.trails.%s' % (trail['HomeRegion'], trail_id) - else: - for key in trail: - trail_config[key] = trail[key] - trail_config['bucket_id'] = self.get_non_provider_id(trail_config.pop('S3BucketName')) - for key in ['IsMultiRegionTrail', 'LogFileValidationEnabled']: - if key not in trail_config: - trail_config[key] = False - trail_details = api_client.get_trail_status(Name=trail['TrailARN']) - for key in ['KMSKeyId', 'IsLogging', 'LatestDeliveryTime', 'LatestDeliveryError', 'StartLoggingTime', - 'StopLoggingTime', 'LatestNotificationTime', 'LatestNotificationError', - 'LatestCloudWatchLogsDeliveryError', 'LatestCloudWatchLogsDeliveryTime']: - trail_config[key] = trail_details[key] if key in trail_details else None - - if trail_details: - # using trail ARN instead of name as with Organizations the trail would be located in another account - trail_config['wildcard_data_logging'] = self.data_logging_status(trail_config['TrailARN'], - trail_details, - api_client) - - for es in api_client.get_event_selectors(TrailName=trail_config['TrailARN'])['EventSelectors']: - trail_config['DataEventsEnabled'] = len(es['DataResources']) > 0 - trail_config['ManagementEventsEnabled'] = es['IncludeManagementEvents'] - - self.trails[trail_id] = trail_config - - def data_logging_status(self, trail_name, trail_details, api_client): - for es in api_client.get_event_selectors(TrailName=trail_name)['EventSelectors']: - has_wildcard = {u'Values': [u'arn:aws:s3:::'], u'Type': u'AWS::S3::Object'} in es['DataResources'] - is_logging = trail_details['IsLogging'] - - if has_wildcard and is_logging and self.is_fresh(trail_details): - return True - - return False - - @staticmethod - def is_fresh(trail_details): - delivery_time = trail_details.get('LatestCloudWatchLogsDeliveryTime', "9999999").strftime("%s") - delivery_age = ((int(time.time()) - int(delivery_time)) / 1440) - return delivery_age <= 24 - - -######################################## -# CloudTrailConfig -######################################## - -class CloudTrailConfig(RegionalServiceConfig): - """ - CloudTrail configuration for all AWS regions - """ - - region_config_class = CloudTrailRegionConfig - - def __init__(self, service_metadata, thread_config=4): - super(CloudTrailConfig, self).__init__(service_metadata, thread_config) - - -######################################## -# Post Processing -######################################## - -def cloudtrail_postprocessing(aws_config): - cloudtrail_config = aws_config['services']['cloudtrail'] - # Global services logging duplicated - if 'cloudtrail-duplicated-global-services-logging' in cloudtrail_config['violations']: - if len(cloudtrail_config['violations']['cloudtrail-duplicated-global-services-logging']['items']) < 2: - cloudtrail_config['violations']['cloudtrail-duplicated-global-services-logging']['items'] = [] - cloudtrail_config['violations']['cloudtrail-duplicated-global-services-logging']['flagged_items'] = 0 - # Global services logging disabled - if 'cloudtrail-no-global-services-logging' in cloudtrail_config['violations']: - if len(cloudtrail_config['violations']['cloudtrail-no-global-services-logging']['items']) != \ - cloudtrail_config['violations']['cloudtrail-no-global-services-logging']['checked_items']: - cloudtrail_config['violations']['cloudtrail-no-global-services-logging']['items'] = [] - cloudtrail_config['violations']['cloudtrail-no-global-services-logging']['flagged_items'] = 0 - # CloudTrail not enabled at all... - if not sum(cloudtrail_config['regions'][r]['trails_count'] for r in cloudtrail_config['regions']): - for r in cloudtrail_config['regions']: - cloudtrail_config['violations']['cloudtrail-no-logging']['items'].append('cloudtrail.regions.%s' % r) - cloudtrail_config['violations']['cloudtrail-no-logging']['checked_items'] += 1 - cloudtrail_config['violations']['cloudtrail-no-logging']['flagged_items'] += 1 - cloudtrail_config['violations']['cloudtrail-no-logging']['dashboard_name'] = 'Regions' From 7dded04efae922554636a7ef312617f6b026c7d0 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 4 Mar 2019 11:06:43 -0500 Subject: [PATCH 195/667] Reverted ctrl+k+d --- ScoutSuite/providers/aws/configs/services.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index 4234c172e..19ea0954b 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -54,24 +54,19 @@ class AWSServicesConfig(BaseServicesConfig): def __init__(self, metadata=None, thread_config=4, **kwargs): super().__init__(metadata, thread_config) - self.cloudformation = CloudFormationConfig( - metadata['management']['cloudformation'], thread_config) + self.cloudformation = CloudFormationConfig(metadata['management']['cloudformation'], thread_config) self.cloudtrail = CloudTrail() - self.cloudwatch = CloudWatchConfig( - metadata['management']['cloudwatch'], thread_config) - self.directconnect = DirectConnectConfig( - metadata['network']['directconnect'], thread_config) + self.cloudwatch = CloudWatchConfig(metadata['management']['cloudwatch'], thread_config) + self.directconnect = DirectConnectConfig(metadata['network']['directconnect'], thread_config) self.ec2 = EC2() self.efs = EFSConfig(metadata['storage']['efs'], thread_config) - self.elasticache = ElastiCacheConfig( - metadata['database']['elasticache'], thread_config) + self.elasticache = ElastiCacheConfig(metadata['database']['elasticache'], thread_config) self.elb = ELBConfig(metadata['compute']['elb'], thread_config) self.elbv2 = ELBv2Config(metadata['compute']['elbv2'], thread_config) self.emr = EMRConfig(metadata['analytics']['emr'], thread_config) self.iam = IAMConfig(thread_config) self.awslambda = Lambdas() - self.redshift = RedshiftConfig( - metadata['database']['redshift'], thread_config) + self.redshift = RedshiftConfig(metadata['database']['redshift'], thread_config) self.rds = RDSConfig(metadata['database']['rds'], thread_config) self.route53 = Route53Config(thread_config) self.route53domains = Route53DomainsConfig(thread_config) From dd993794ebc4de68b12984f0ba7817c5badda831 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 4 Mar 2019 11:09:54 -0500 Subject: [PATCH 196/667] Make fetch_all and finalize methods async. --- ScoutSuite/providers/base/configs/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/base/configs/base.py b/ScoutSuite/providers/base/configs/base.py index 237fefc19..68e427167 100644 --- a/ScoutSuite/providers/base/configs/base.py +++ b/ScoutSuite/providers/base/configs/base.py @@ -63,7 +63,7 @@ def get_non_provider_id(self, name): m.update(name.encode('utf-8')) return m.hexdigest() - def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): + async def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): """ :param credentials: F :param service: Name of the service @@ -230,7 +230,7 @@ def _get_targets(self, response_attribute, api_client, method, list_params, igno """ return None - def finalize(self): + async def finalize(self): for t in self.fetchstatuslogger.counts: setattr(self, '%s_count' % t, self.fetchstatuslogger.counts[t]['fetched']) self.__delattr__('fetchstatuslogger') From 35a4147c1ad322a806ce5805da90158c4d206913 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 4 Mar 2019 11:10:30 -0500 Subject: [PATCH 197/667] Remove encoding statement at the beginning of each file. --- ScoutSuite/providers/azure/resources/resources.py | 2 -- .../azure/resources/sql/database_blob_auditing_policies.py | 2 -- .../azure/resources/sql/database_threat_detection_policies.py | 2 -- ScoutSuite/providers/azure/resources/sql/databases.py | 2 -- ScoutSuite/providers/azure/resources/sql/replication_links.py | 2 -- .../azure/resources/sql/server_azure_ad_administrators.py | 2 -- ScoutSuite/providers/azure/resources/sql/servers.py | 2 -- .../azure/resources/sql/transparent_data_encryptions.py | 2 -- 8 files changed, 16 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/resources.py b/ScoutSuite/providers/azure/resources/resources.py index c802dc13a..563c25ca7 100644 --- a/ScoutSuite/providers/azure/resources/resources.py +++ b/ScoutSuite/providers/azure/resources/resources.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import abc from ScoutSuite.providers.base.configs.resources import CompositeResources diff --git a/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py b/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py index 2e0b2f66f..cf9d9dda5 100644 --- a/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py +++ b/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - from ScoutSuite.providers.base.configs.resources import Resources diff --git a/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py b/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py index 82a499f3b..a758d4496 100644 --- a/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py +++ b/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - from ScoutSuite.providers.base.configs.resources import Resources diff --git a/ScoutSuite/providers/azure/resources/sql/databases.py b/ScoutSuite/providers/azure/resources/sql/databases.py index 0bd01e2a5..3bd0f807b 100644 --- a/ScoutSuite/providers/azure/resources/sql/databases.py +++ b/ScoutSuite/providers/azure/resources/sql/databases.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - from ..resources import AzureCompositeResources from .database_blob_auditing_policies import DatabaseBlobAuditingPolicies diff --git a/ScoutSuite/providers/azure/resources/sql/replication_links.py b/ScoutSuite/providers/azure/resources/sql/replication_links.py index bdd8323a9..57f5bab72 100644 --- a/ScoutSuite/providers/azure/resources/sql/replication_links.py +++ b/ScoutSuite/providers/azure/resources/sql/replication_links.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - from ScoutSuite.providers.base.configs.resources import Resources diff --git a/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py b/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py index 7666757e0..fc67436a9 100644 --- a/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py +++ b/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - from msrestazure.azure_exceptions import CloudError from ScoutSuite.providers.base.configs.resources import Resources diff --git a/ScoutSuite/providers/azure/resources/sql/servers.py b/ScoutSuite/providers/azure/resources/sql/servers.py index 38fd1b256..eede22bf0 100644 --- a/ScoutSuite/providers/azure/resources/sql/servers.py +++ b/ScoutSuite/providers/azure/resources/sql/servers.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - from azure.mgmt.sql import SqlManagementClient from ..resources import AzureCompositeResources diff --git a/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py b/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py index 96e64efa1..b4aa4c076 100644 --- a/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py +++ b/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - from ScoutSuite.providers.base.configs.resources import Resources From 64e0d336fd13cd86a832e13b392fc7bf024f4b11 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 4 Mar 2019 11:18:53 -0500 Subject: [PATCH 198/667] Update get_non_provider_id and move utils.py up. --- ScoutSuite/providers/azure/resources/sql/servers.py | 2 +- ScoutSuite/providers/{azure/resources => }/utils.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) rename ScoutSuite/providers/{azure/resources => }/utils.py (79%) diff --git a/ScoutSuite/providers/azure/resources/sql/servers.py b/ScoutSuite/providers/azure/resources/sql/servers.py index eede22bf0..a6c6a4772 100644 --- a/ScoutSuite/providers/azure/resources/sql/servers.py +++ b/ScoutSuite/providers/azure/resources/sql/servers.py @@ -5,7 +5,7 @@ from .databases import Databases from .server_azure_ad_administrators import ServerAzureAdAdministrators -from ..utils import get_non_provider_id +from ScoutSuite.providers.utils import get_non_provider_id class Servers(AzureCompositeResources): diff --git a/ScoutSuite/providers/azure/resources/utils.py b/ScoutSuite/providers/utils.py similarity index 79% rename from ScoutSuite/providers/azure/resources/utils.py rename to ScoutSuite/providers/utils.py index d1340ae97..dfbf6e514 100644 --- a/ScoutSuite/providers/azure/resources/utils.py +++ b/ScoutSuite/providers/utils.py @@ -9,6 +9,6 @@ def get_non_provider_id(name): :param name: Name of the resource to :return: SHA1(name) """ - m = sha1() - m.update(name.encode('utf-8')) - return m.hexdigest() + _hash = sha1() + _hash.update(name.encode('utf-8')) + return _hash.hexdigest() From 08b1ad542f58e1b18199a019f9ebf363b0d60228 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 4 Mar 2019 11:28:14 -0500 Subject: [PATCH 199/667] Rename sql dir to sqldatabase to match service name. --- .../providers/azure/resources/{sql => sqldatabase}/__init__.py | 0 .../{sql => sqldatabase}/database_blob_auditing_policies.py | 0 .../{sql => sqldatabase}/database_threat_detection_policies.py | 0 .../providers/azure/resources/{sql => sqldatabase}/databases.py | 0 .../azure/resources/{sql => sqldatabase}/replication_links.py | 0 .../{sql => sqldatabase}/server_azure_ad_administrators.py | 0 .../providers/azure/resources/{sql => sqldatabase}/servers.py | 0 .../{sql => sqldatabase}/transparent_data_encryptions.py | 0 8 files changed, 0 insertions(+), 0 deletions(-) rename ScoutSuite/providers/azure/resources/{sql => sqldatabase}/__init__.py (100%) rename ScoutSuite/providers/azure/resources/{sql => sqldatabase}/database_blob_auditing_policies.py (100%) rename ScoutSuite/providers/azure/resources/{sql => sqldatabase}/database_threat_detection_policies.py (100%) rename ScoutSuite/providers/azure/resources/{sql => sqldatabase}/databases.py (100%) rename ScoutSuite/providers/azure/resources/{sql => sqldatabase}/replication_links.py (100%) rename ScoutSuite/providers/azure/resources/{sql => sqldatabase}/server_azure_ad_administrators.py (100%) rename ScoutSuite/providers/azure/resources/{sql => sqldatabase}/servers.py (100%) rename ScoutSuite/providers/azure/resources/{sql => sqldatabase}/transparent_data_encryptions.py (100%) diff --git a/ScoutSuite/providers/azure/resources/sql/__init__.py b/ScoutSuite/providers/azure/resources/sqldatabase/__init__.py similarity index 100% rename from ScoutSuite/providers/azure/resources/sql/__init__.py rename to ScoutSuite/providers/azure/resources/sqldatabase/__init__.py diff --git a/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py b/ScoutSuite/providers/azure/resources/sqldatabase/database_blob_auditing_policies.py similarity index 100% rename from ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py rename to ScoutSuite/providers/azure/resources/sqldatabase/database_blob_auditing_policies.py diff --git a/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py b/ScoutSuite/providers/azure/resources/sqldatabase/database_threat_detection_policies.py similarity index 100% rename from ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py rename to ScoutSuite/providers/azure/resources/sqldatabase/database_threat_detection_policies.py diff --git a/ScoutSuite/providers/azure/resources/sql/databases.py b/ScoutSuite/providers/azure/resources/sqldatabase/databases.py similarity index 100% rename from ScoutSuite/providers/azure/resources/sql/databases.py rename to ScoutSuite/providers/azure/resources/sqldatabase/databases.py diff --git a/ScoutSuite/providers/azure/resources/sql/replication_links.py b/ScoutSuite/providers/azure/resources/sqldatabase/replication_links.py similarity index 100% rename from ScoutSuite/providers/azure/resources/sql/replication_links.py rename to ScoutSuite/providers/azure/resources/sqldatabase/replication_links.py diff --git a/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py b/ScoutSuite/providers/azure/resources/sqldatabase/server_azure_ad_administrators.py similarity index 100% rename from ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py rename to ScoutSuite/providers/azure/resources/sqldatabase/server_azure_ad_administrators.py diff --git a/ScoutSuite/providers/azure/resources/sql/servers.py b/ScoutSuite/providers/azure/resources/sqldatabase/servers.py similarity index 100% rename from ScoutSuite/providers/azure/resources/sql/servers.py rename to ScoutSuite/providers/azure/resources/sqldatabase/servers.py diff --git a/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py b/ScoutSuite/providers/azure/resources/sqldatabase/transparent_data_encryptions.py similarity index 100% rename from ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py rename to ScoutSuite/providers/azure/resources/sqldatabase/transparent_data_encryptions.py From 123b5b46aec5c6a34cf4170fcfdf1dec6c0137f9 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 4 Mar 2019 11:33:32 -0500 Subject: [PATCH 200/667] Deleted commented code --- .../aws/resources/cloudtrail/service.py | 107 ------------------ 1 file changed, 107 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/cloudtrail/service.py b/ScoutSuite/providers/aws/resources/cloudtrail/service.py index 381b57de6..826d014ff 100644 --- a/ScoutSuite/providers/aws/resources/cloudtrail/service.py +++ b/ScoutSuite/providers/aws/resources/cloudtrail/service.py @@ -68,110 +68,3 @@ class CloudTrail(Regions): def __init__(self): super(CloudTrail, self).__init__('cloudtrail') - - -# # -*- coding: utf-8 -*- -# """ -# CloudTrail-related classes and functions -# """ -# import time - -# from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients - - -# ######################################## -# # CloudTrailRegionConfig -# ######################################## - -# class CloudTrailRegionConfig(RegionConfig): -# """ -# CloudTrail configuration for a single AWS region -# """ -# trails = {} - -# def parse_trail(self, global_params, region, trail): -# """ -# Parse a single CloudTrail trail - -# :param trail: -# :param global_params: Parameters shared for all regions -# :param region: Name of the AWS region -# """ -# trail_config = {'name': trail.pop('Name')} -# trail_id = self.get_non_provider_id(trail_config['name']) -# trail_details = None - -# api_client = api_clients[region] - -# # Do not duplicate entries for multiregion trails -# if 'IsMultiRegionTrail' in trail and trail['IsMultiRegionTrail'] and trail['HomeRegion'] != region: -# for key in ['HomeRegion', 'TrailARN']: -# trail_config[key] = trail[key] -# trail_config['scout2_link'] = 'services.cloudtrail.regions.%s.trails.%s' % (trail['HomeRegion'], trail_id) -# else: -# for key in trail: -# trail_config[key] = trail[key] -# trail_config['bucket_id'] = self.get_non_provider_id(trail_config.pop('S3BucketName')) -# for key in ['IsMultiRegionTrail', 'LogFileValidationEnabled']: -# if key not in trail_config: -# trail_config[key] = False -# trail_details = api_client.get_trail_status(Name=trail['TrailARN']) -# for key in ['KMSKeyId', 'IsLogging', 'LatestDeliveryTime', 'LatestDeliveryError', 'StartLoggingTime', -# 'StopLoggingTime', 'LatestNotificationTime', 'LatestNotificationError', -# 'LatestCloudWatchLogsDeliveryError', 'LatestCloudWatchLogsDeliveryTime']: -# trail_config[key] = trail_details[key] if key in trail_details else None - -# if trail_details: -# # using trail ARN instead of name as with Organizations the trail would be located in another account -# trail_config['wildcard_data_logging'] = self.data_logging_status(trail_config['TrailARN'], -# trail_details, -# api_client) - -# for es in api_client.get_event_selectors(TrailName=trail_config['TrailARN'])['EventSelectors']: -# trail_config['DataEventsEnabled'] = len(es['DataResources']) > 0 -# trail_config['ManagementEventsEnabled'] = es['IncludeManagementEvents'] - -# self.trails[trail_id] = trail_config - - - - -# ######################################## -# # CloudTrailConfig -# ######################################## - -# class CloudTrailConfig(RegionalServiceConfig): -# """ -# CloudTrail configuration for all AWS regions -# """ - -# region_config_class = CloudTrailRegionConfig - -# def __init__(self, service_metadata, thread_config=4): -# super(CloudTrailConfig, self).__init__(service_metadata, thread_config) - - -# ######################################## -# # Post Processing -# ######################################## - -# def cloudtrail_postprocessing(aws_config): -# cloudtrail_config = aws_config['services']['cloudtrail'] -# # Global services logging duplicated -# if 'cloudtrail-duplicated-global-services-logging' in cloudtrail_config['violations']: -# if len(cloudtrail_config['violations']['cloudtrail-duplicated-global-services-logging']['items']) < 2: -# cloudtrail_config['violations']['cloudtrail-duplicated-global-services-logging']['items'] = [] -# cloudtrail_config['violations']['cloudtrail-duplicated-global-services-logging']['flagged_items'] = 0 -# # Global services logging disabled -# if 'cloudtrail-no-global-services-logging' in cloudtrail_config['violations']: -# if len(cloudtrail_config['violations']['cloudtrail-no-global-services-logging']['items']) != \ -# cloudtrail_config['violations']['cloudtrail-no-global-services-logging']['checked_items']: -# cloudtrail_config['violations']['cloudtrail-no-global-services-logging']['items'] = [] -# cloudtrail_config['violations']['cloudtrail-no-global-services-logging']['flagged_items'] = 0 -# # CloudTrail not enabled at all... -# if not sum(cloudtrail_config['regions'][r]['trails_count'] for r in cloudtrail_config['regions']): -# for r in cloudtrail_config['regions']: -# cloudtrail_config['violations']['cloudtrail-no-logging']['items'].append('cloudtrail.regions.%s' % r) -# cloudtrail_config['violations']['cloudtrail-no-logging']['checked_items'] += 1 -# cloudtrail_config['violations']['cloudtrail-no-logging']['flagged_items'] += 1 -# cloudtrail_config['violations']['cloudtrail-no-logging']['dashboard_name'] = 'Regions' From ed77cd91239e81b4f800e9479dc01ab35d93d462 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 4 Mar 2019 11:42:18 -0500 Subject: [PATCH 201/667] Use absolute paths instead of .. --- ScoutSuite/providers/azure/resources/sqldatabase/databases.py | 2 +- ScoutSuite/providers/azure/resources/sqldatabase/servers.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sqldatabase/databases.py b/ScoutSuite/providers/azure/resources/sqldatabase/databases.py index 3bd0f807b..f2a28dae0 100644 --- a/ScoutSuite/providers/azure/resources/sqldatabase/databases.py +++ b/ScoutSuite/providers/azure/resources/sqldatabase/databases.py @@ -1,4 +1,4 @@ -from ..resources import AzureCompositeResources +from ScoutSuite.providers.azure.resources.resources import AzureCompositeResources from .database_blob_auditing_policies import DatabaseBlobAuditingPolicies from .database_threat_detection_policies import DatabaseThreatDetectionPolicies diff --git a/ScoutSuite/providers/azure/resources/sqldatabase/servers.py b/ScoutSuite/providers/azure/resources/sqldatabase/servers.py index a6c6a4772..595298607 100644 --- a/ScoutSuite/providers/azure/resources/sqldatabase/servers.py +++ b/ScoutSuite/providers/azure/resources/sqldatabase/servers.py @@ -1,11 +1,11 @@ from azure.mgmt.sql import SqlManagementClient -from ..resources import AzureCompositeResources +from ScoutSuite.providers.azure.resources.resources import AzureCompositeResources from ScoutSuite.providers.azure.utils import get_resource_group_name +from ScoutSuite.providers.utils import get_non_provider_id from .databases import Databases from .server_azure_ad_administrators import ServerAzureAdAdministrators -from ScoutSuite.providers.utils import get_non_provider_id class Servers(AzureCompositeResources): From 0737ec39001d3752bdbe10c876be6b69e021c039 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 4 Mar 2019 12:01:43 -0500 Subject: [PATCH 202/667] Fix import. --- ScoutSuite/providers/azure/configs/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/azure/configs/services.py b/ScoutSuite/providers/azure/configs/services.py index c5520dcfd..9bf5da7fc 100644 --- a/ScoutSuite/providers/azure/configs/services.py +++ b/ScoutSuite/providers/azure/configs/services.py @@ -3,7 +3,7 @@ from ScoutSuite.providers.base.configs.services import BaseServicesConfig from ScoutSuite.providers.azure.services.storageaccounts import StorageAccountsConfig from ScoutSuite.providers.azure.services.monitor import MonitorConfig -from ScoutSuite.providers.azure.resources.sql.servers import Servers as SQLDatabaseConfig +from ScoutSuite.providers.azure.resources.sqldatabase.servers import Servers as SQLDatabaseConfig from ScoutSuite.providers.azure.services.securitycenter import SecurityCenterConfig from ScoutSuite.providers.azure.services.network import NetworkConfig from ScoutSuite.providers.azure.services.keyvault import KeyVaultConfig From 40e83a4c0d9d86bd786e65765004f2ac5946dd93 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 4 Mar 2019 12:30:39 -0500 Subject: [PATCH 203/667] Fix loop assignment. --- tests/test-aws_resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test-aws_resources.py b/tests/test-aws_resources.py index eebea04ab..1c85e156b 100644 --- a/tests/test-aws_resources.py +++ b/tests/test-aws_resources.py @@ -32,7 +32,7 @@ class TestAWSResources(TestCase): test_dir = os.path.dirname(os.path.realpath(__file__)) def test_lel(self): - loop = loop = asyncio.new_event_loop() + loop = asyncio.new_event_loop() composite = DummyComposite({'region': 'some_region'}) loop.run_until_complete(composite.fetch_all()) From 45ad543e8f0771e02291c1eb2171d7d53744b78c Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 4 Mar 2019 14:12:33 -0500 Subject: [PATCH 204/667] Parse retention config. --- .../resources/sqldatabase/database_blob_auditing_policies.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/providers/azure/resources/sqldatabase/database_blob_auditing_policies.py b/ScoutSuite/providers/azure/resources/sqldatabase/database_blob_auditing_policies.py index cf9d9dda5..dd28d89f4 100644 --- a/ScoutSuite/providers/azure/resources/sqldatabase/database_blob_auditing_policies.py +++ b/ScoutSuite/providers/azure/resources/sqldatabase/database_blob_auditing_policies.py @@ -17,5 +17,6 @@ async def fetch_all(self): def _parse_policies(self, policies): self.update({ - 'auditing_enabled': policies.state == "Enabled" + 'auditing_enabled': policies.state == "Enabled", + 'retention_days': policies.retention_days }) From 5ff0eff9e5e2148ea59de27eea713ea2cc952bb1 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 4 Mar 2019 14:13:11 -0500 Subject: [PATCH 205/667] Parse alerts and retention config. --- .../sqldatabase/database_threat_detection_policies.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/providers/azure/resources/sqldatabase/database_threat_detection_policies.py b/ScoutSuite/providers/azure/resources/sqldatabase/database_threat_detection_policies.py index a758d4496..424ed2c1f 100644 --- a/ScoutSuite/providers/azure/resources/sqldatabase/database_threat_detection_policies.py +++ b/ScoutSuite/providers/azure/resources/sqldatabase/database_threat_detection_policies.py @@ -17,5 +17,8 @@ async def fetch_all(self): def _parse_policies(self, policies): self.update({ - 'threat_detection_enabled': policies.state == "Enabled" + 'threat_detection_enabled': policies.state == "Enabled", + 'alerts_enabled': policies.disabled_alerts == "", + 'send_alerts_enabled': policies.email_addresses != "" and policies.email_account_admins == "Enabled", + 'retention_days': policies.retention_days }) From 2b36163f692d146fd10c897675059effaa8e686a Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 4 Mar 2019 14:14:05 -0500 Subject: [PATCH 206/667] Parse auditing config for sql servers. --- .../server_blob_auditing_policies.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 ScoutSuite/providers/azure/resources/sqldatabase/server_blob_auditing_policies.py diff --git a/ScoutSuite/providers/azure/resources/sqldatabase/server_blob_auditing_policies.py b/ScoutSuite/providers/azure/resources/sqldatabase/server_blob_auditing_policies.py new file mode 100644 index 000000000..7a0a5ff55 --- /dev/null +++ b/ScoutSuite/providers/azure/resources/sqldatabase/server_blob_auditing_policies.py @@ -0,0 +1,21 @@ +from ScoutSuite.providers.base.configs.resources import Resources + + +class ServerBlobAuditingPolicies(Resources): + + def __init__(self, resource_group_name, server_name, facade): + self.resource_group_name = resource_group_name + self.server_name = server_name + self.facade = facade + + # TODO: make it really async. + async def fetch_all(self): + policies = self.facade.server_blob_auditing_policies.get( + self.resource_group_name, self.server_name) + self._parse_policies(policies) + + def _parse_policies(self, policies): + self.update({ + 'auditing_enabled': policies.state == "Enabled", + 'retention_days': policies.retention_days + }) From a3fc953f4ec4eaf9a5695c4999c14c11c8520bbd Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 4 Mar 2019 14:14:35 -0500 Subject: [PATCH 207/667] Parse security config for sql servers. --- .../server_security_alert_policies.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 ScoutSuite/providers/azure/resources/sqldatabase/server_security_alert_policies.py diff --git a/ScoutSuite/providers/azure/resources/sqldatabase/server_security_alert_policies.py b/ScoutSuite/providers/azure/resources/sqldatabase/server_security_alert_policies.py new file mode 100644 index 000000000..59ef96d2a --- /dev/null +++ b/ScoutSuite/providers/azure/resources/sqldatabase/server_security_alert_policies.py @@ -0,0 +1,23 @@ +from ScoutSuite.providers.base.configs.resources import Resources + + +class ServerSecurityAlertPolicies(Resources): + + def __init__(self, resource_group_name, server_name, facade): + self.resource_group_name = resource_group_name + self.server_name = server_name + self.facade = facade + + # TODO: make it really async. + async def fetch_all(self): + policies = self.facade.server_security_alert_policies.get( + self.resource_group_name, self.server_name) + self._parse_policies(policies) + + def _parse_policies(self, policies): + self.update({ + 'threat_detection_enabled': policies.state == "Enabled", + 'alerts_enabled': policies.disabled_alerts == [""], + 'send_alerts_enabled': policies.email_addresses != [""] and policies.email_account_admins, + 'retention_days': policies.retention_days + }) From 903028ba4135da956e1cd6ba514133e83242d110 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 4 Mar 2019 14:15:45 -0500 Subject: [PATCH 208/667] Add support for children names in AzureCompositeResources. --- ScoutSuite/providers/azure/resources/resources.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/resources.py b/ScoutSuite/providers/azure/resources/resources.py index 563c25ca7..6ea29637c 100644 --- a/ScoutSuite/providers/azure/resources/resources.py +++ b/ScoutSuite/providers/azure/resources/resources.py @@ -3,10 +3,14 @@ from ScoutSuite.providers.base.configs.resources import CompositeResources +# TODO: add docstrings. class AzureCompositeResources(CompositeResources, metaclass=abc.ABCMeta): async def _fetch_children(self, parent, **kwargs): - for child_class in self._children: + for child_class, name in self._children: child = child_class(**kwargs) await child.fetch_all() - parent.update(child) + if name: + parent[name] = child + else: + parent.update(child) From 14305b1f1455308571eee54eb5b92a72d99386e1 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 4 Mar 2019 14:16:18 -0500 Subject: [PATCH 209/667] Add children names. --- .../azure/resources/sqldatabase/databases.py | 14 ++++++-------- .../azure/resources/sqldatabase/servers.py | 8 ++++++-- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sqldatabase/databases.py b/ScoutSuite/providers/azure/resources/sqldatabase/databases.py index f2a28dae0..81f02e272 100644 --- a/ScoutSuite/providers/azure/resources/sqldatabase/databases.py +++ b/ScoutSuite/providers/azure/resources/sqldatabase/databases.py @@ -8,10 +8,10 @@ class Databases(AzureCompositeResources): _children = [ - DatabaseBlobAuditingPolicies, - DatabaseThreatDetectionPolicies, - ReplicationLinks, - TransparentDataEncryptions, + (DatabaseBlobAuditingPolicies, 'auditing'), + (DatabaseThreatDetectionPolicies, 'threat_detection'), + (ReplicationLinks, None), + (TransparentDataEncryptions, None) ] def __init__(self, resource_group_name, server_name, facade): @@ -21,18 +21,16 @@ def __init__(self, resource_group_name, server_name, facade): # TODO: make it really async. async def fetch_all(self): - self['databases'] = {} - for db in self.facade.databases.list_by_server(self.resource_group_name, self.server_name): # We do not want to scan 'master' database which is auto-generated by Azure and read-only: if db.name == 'master': continue - self['databases'][db.name] = { + self[db.name] = { 'id': db.name, } await self._fetch_children( - parent=self['databases'][db.name], + parent=self[db.name], resource_group_name=self.resource_group_name, server_name=self.server_name, database_name=db.name, diff --git a/ScoutSuite/providers/azure/resources/sqldatabase/servers.py b/ScoutSuite/providers/azure/resources/sqldatabase/servers.py index 595298607..1408edeaa 100644 --- a/ScoutSuite/providers/azure/resources/sqldatabase/servers.py +++ b/ScoutSuite/providers/azure/resources/sqldatabase/servers.py @@ -6,12 +6,16 @@ from .databases import Databases from .server_azure_ad_administrators import ServerAzureAdAdministrators +from .server_blob_auditing_policies import ServerBlobAuditingPolicies +from .server_security_alert_policies import ServerSecurityAlertPolicies class Servers(AzureCompositeResources): _children = [ - Databases, - ServerAzureAdAdministrators, + (Databases, 'databases'), + (ServerAzureAdAdministrators, None), + (ServerBlobAuditingPolicies, 'auditing'), + (ServerSecurityAlertPolicies, 'threat_detection') ] # TODO: make it really async. From 6459856a2bb48370c4671f65fcb59f98a3bab038 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 4 Mar 2019 14:20:47 -0500 Subject: [PATCH 210/667] Remove ghost dir. --- .../providers/azure/resources/sql/__init__.py | 0 .../sql/database_blob_auditing_policies.py | 22 ---------- .../sql/database_threat_detection_policies.py | 22 ---------- .../azure/resources/sql/databases.py | 42 ------------------- .../azure/resources/sql/replication_links.py | 22 ---------- .../sql/server_azure_ad_administrators.py | 21 ---------- .../sql/server_blob_auditing_policies.py | 21 ---------- .../providers/azure/resources/sql/servers.py | 41 ------------------ .../sql/transparent_data_encryptions.py | 22 ---------- 9 files changed, 213 deletions(-) delete mode 100644 ScoutSuite/providers/azure/resources/sql/__init__.py delete mode 100644 ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py delete mode 100644 ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py delete mode 100644 ScoutSuite/providers/azure/resources/sql/databases.py delete mode 100644 ScoutSuite/providers/azure/resources/sql/replication_links.py delete mode 100644 ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py delete mode 100644 ScoutSuite/providers/azure/resources/sql/server_blob_auditing_policies.py delete mode 100644 ScoutSuite/providers/azure/resources/sql/servers.py delete mode 100644 ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py diff --git a/ScoutSuite/providers/azure/resources/sql/__init__.py b/ScoutSuite/providers/azure/resources/sql/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py b/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py deleted file mode 100644 index 637eae5dd..000000000 --- a/ScoutSuite/providers/azure/resources/sql/database_blob_auditing_policies.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- - -from ..resources import AzureSimpleResources - - -class DatabaseBlobAuditingPolicies(AzureSimpleResources): - - def __init__(self, resource_group_name, server_name, database_name, facade): - self.resource_group_name = resource_group_name - self.server_name = server_name - self.database_name = database_name - self.facade = facade - - # TODO: make it really async. - async def get_resources_from_api(self): - return self.facade.database_blob_auditing_policies.get( - self.resource_group_name, self.server_name, self.database_name) - - def parse_resource(self, policies): - self.update({ - 'auditing_enabled': policies.state == "Enabled" - }) diff --git a/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py b/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py deleted file mode 100644 index 84977378f..000000000 --- a/ScoutSuite/providers/azure/resources/sql/database_threat_detection_policies.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- - -from ..resources import AzureSimpleResources - - -class DatabaseThreatDetectionPolicies(AzureSimpleResources): - - def __init__(self, resource_group_name, server_name, database_name, facade): - self.resource_group_name = resource_group_name - self.server_name = server_name - self.database_name = database_name - self.facade = facade - - # TODO: make it really async. - async def get_resources_from_api(self): - return self.facade.database_threat_detection_policies.get( - self.resource_group_name, self.server_name, self.database_name) - - def parse_resource(self, policies): - self.update({ - 'threat_detection_enabled': policies.state == "Enabled" - }) diff --git a/ScoutSuite/providers/azure/resources/sql/databases.py b/ScoutSuite/providers/azure/resources/sql/databases.py deleted file mode 100644 index 4de894a47..000000000 --- a/ScoutSuite/providers/azure/resources/sql/databases.py +++ /dev/null @@ -1,42 +0,0 @@ -# -*- coding: utf-8 -*- - -from ..resources import AzureCompositeResources - -from .database_blob_auditing_policies import DatabaseBlobAuditingPolicies -from .database_threat_detection_policies import DatabaseThreatDetectionPolicies -from .replication_links import ReplicationLinks -from .transparent_data_encryptions import TransparentDataEncryptions - - -class Databases(AzureCompositeResources): - children = [ - DatabaseBlobAuditingPolicies, - DatabaseThreatDetectionPolicies, - ReplicationLinks, - TransparentDataEncryptions, - ] - - def __init__(self, resource_group_name, server_name, facade): - self.resource_group_name = resource_group_name - self.server_name = server_name - self.facade = facade - - # TODO: make it really async. - async def fetch_all(self): - self['databases'] = {} - - for db in self.facade.databases.list_by_server(self.resource_group_name, self.server_name): - # We do not want to scan 'master' database which is auto-generated by Azure and read-only: - if db.name == 'master': - continue - - self['databases'][db.name] = { - 'id': db.name, - } - await self.fetch_children( - parent=self['databases'][db.name], - resource_group_name=self.resource_group_name, - server_name=self.server_name, - database_name=db.name, - facade=self.facade - ) diff --git a/ScoutSuite/providers/azure/resources/sql/replication_links.py b/ScoutSuite/providers/azure/resources/sql/replication_links.py deleted file mode 100644 index f15e7e27c..000000000 --- a/ScoutSuite/providers/azure/resources/sql/replication_links.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- - -from ..resources import AzureSimpleResources - - -class ReplicationLinks(AzureSimpleResources): - - def __init__(self, resource_group_name, server_name, database_name, facade): - self.resource_group_name = resource_group_name - self.server_name = server_name - self.database_name = database_name - self.facade = facade - - # TODO: make it really async. - async def get_resources_from_api(self): - return list(self.facade.replication_links.list_by_database( - self.resource_group_name, self.server_name, self.database_name)) - - def parse_resource(self, links): - self.update({ - 'replication_configured': len(links) > 0 - }) diff --git a/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py b/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py deleted file mode 100644 index 7666757e0..000000000 --- a/ScoutSuite/providers/azure/resources/sql/server_azure_ad_administrators.py +++ /dev/null @@ -1,21 +0,0 @@ -# -*- coding: utf-8 -*- - -from msrestazure.azure_exceptions import CloudError - -from ScoutSuite.providers.base.configs.resources import Resources - - -class ServerAzureAdAdministrators(Resources): - - def __init__(self, resource_group_name, server_name, facade): - self.resource_group_name = resource_group_name - self.server_name = server_name - self.facade = facade - - # TODO: make it really async. - async def fetch_all(self): - try: - self.facade.server_azure_ad_administrators.get(self.resource_group_name, self.server_name) - self['ad_admin_configured'] = True - except CloudError: # no ad admin configured returns a 404 error - self['ad_admin_configured'] = False diff --git a/ScoutSuite/providers/azure/resources/sql/server_blob_auditing_policies.py b/ScoutSuite/providers/azure/resources/sql/server_blob_auditing_policies.py deleted file mode 100644 index 606f00c8d..000000000 --- a/ScoutSuite/providers/azure/resources/sql/server_blob_auditing_policies.py +++ /dev/null @@ -1,21 +0,0 @@ -# -*- coding: utf-8 -*- - -from ..resources import AzureSimpleResources - - -class ServerBlobAuditingPolicies(AzureSimpleResources): - - def __init__(self, resource_group_name, server_name, facade): - self.resource_group_name = resource_group_name - self.server_name = server_name - self.facade = facade - - # TODO: make it really async. - async def get_resources_from_api(self): - return self.facade.server_blob_auditing_policies.get( - self.resource_group_name, self.server_name) - - def parse_resource(self, policies): - self.update({ - 'auditing_enabled': policies.state == "Enabled" - }) diff --git a/ScoutSuite/providers/azure/resources/sql/servers.py b/ScoutSuite/providers/azure/resources/sql/servers.py deleted file mode 100644 index b9635db1d..000000000 --- a/ScoutSuite/providers/azure/resources/sql/servers.py +++ /dev/null @@ -1,41 +0,0 @@ -# -*- coding: utf-8 -*- - -from azure.mgmt.sql import SqlManagementClient - -from ..resources import AzureCompositeResources -from ScoutSuite.providers.azure.utils import get_resource_group_name - -from .databases import Databases -from .server_azure_ad_administrators import ServerAzureAdAdministrators -from .server_blob_auditing_policies import ServerBlobAuditingPolicies -from ..utils import get_non_provider_id - - -class Servers(AzureCompositeResources): - children = [ - Databases, - ServerAzureAdAdministrators, - ServerBlobAuditingPolicies - ] - - # TODO: make it really async. - async def fetch_all(self, credentials, **kwargs): - # TODO: build that facade somewhere else: - facade = SqlManagementClient(credentials.credentials, credentials.subscription_id) - - self['servers'] = {} - for server in facade.servers.list(): - id = get_non_provider_id(server.id) - resource_group_name = get_resource_group_name(server.id) - - self['servers'][id] = { - 'id': id, - 'name': server.name - } - await self.fetch_children( - parent=self['servers'][id], - resource_group_name=resource_group_name, - server_name=server.name, - facade=facade) - - self['servers_count'] = len(self['servers']) diff --git a/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py b/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py deleted file mode 100644 index d82b51fce..000000000 --- a/ScoutSuite/providers/azure/resources/sql/transparent_data_encryptions.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- - -from ..resources import AzureSimpleResources - - -class TransparentDataEncryptions(AzureSimpleResources): - - def __init__(self, resource_group_name, server_name, database_name, facade): - self.resource_group_name = resource_group_name - self.server_name = server_name - self.database_name = database_name - self.facade = facade - - # TODO: make it really async. - async def get_resources_from_api(self): - return self.facade.transparent_data_encryptions.get( - self.resource_group_name, self.server_name, self.database_name) - - def parse_resource(self, encryptions): - return self.update({ - 'transparent_data_encryption_enabled': encryptions.status == "Enabled" - }) From da883021c758147af3b1d43bf92120f395b7b70a Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 4 Mar 2019 14:21:46 -0500 Subject: [PATCH 211/667] Add new rule. --- .../sqldatabase-databases-auditing-low-retention.json | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-auditing-low-retention.json diff --git a/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-auditing-low-retention.json b/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-auditing-low-retention.json new file mode 100644 index 000000000..f8734d2cc --- /dev/null +++ b/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-auditing-low-retention.json @@ -0,0 +1,11 @@ +{ + "dashboard_name": "SQL Databases", + "description": "Auditing retention period for some SQL databases is too short", + "rationale": "Auditing retention period should be greater than _ARG_0_ days (CIS 4.2.7).", + "path": "sqldatabase.servers.id.databases.id", + "display_path": "sqldatabase.servers.id", + "conditions": [ "and", + [ "sqldatabase.servers.id.databases.id.auditing.retention_days", "lessThan", "_ARG_0_" ] + ], + "id_suffix": "db_low_auditing_retention" +} \ No newline at end of file From 18fc7a5c20283cc187f5da0274b942d67f01fa12 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 4 Mar 2019 14:22:20 -0500 Subject: [PATCH 212/667] Update rule. --- .../rules/findings/sqldatabase-databases-no-auditing.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-no-auditing.json b/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-no-auditing.json index 55fd65d10..dcfaf13b3 100644 --- a/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-no-auditing.json +++ b/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-no-auditing.json @@ -1,11 +1,11 @@ { "dashboard_name": "SQL Databases", - "description": "Auditing disabled on SQL databases", + "description": "Auditing is disabled for some SQL databases", "rationale": "You should enable auditing for all of your SQL databases (CIS 4.2.1).", "path": "sqldatabase.servers.id.databases.id", "display_path": "sqldatabase.servers.id", "conditions": [ "and", - [ "sqldatabase.servers.id.databases.id.auditing_enabled", "false", "" ] + [ "sqldatabase.servers.id.databases.id.auditing.auditing_enabled", "false", "" ] ], - "id_suffix": "auditing_enabled" + "id_suffix": "db_auditing_disabled" } \ No newline at end of file From 29379de594a697f0d8c184d8a94c3a986656c3a2 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 4 Mar 2019 14:22:35 -0500 Subject: [PATCH 213/667] Update rule. --- .../sqldatabase-databases-no-threat-detection.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-no-threat-detection.json b/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-no-threat-detection.json index 29f72911f..7da59a7f7 100644 --- a/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-no-threat-detection.json +++ b/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-no-threat-detection.json @@ -1,11 +1,11 @@ { "dashboard_name": "SQL Databases", - "description": "Threat detection disabled", - "rationale": "You should enable threat detection for all of your SQL databases. See CIS 4.2.2.", + "description": "Threat detection is disabled for some SQL databases", + "rationale": "You should enable threat detection for all of your SQL databases (CIS 4.2.2).", "path": "sqldatabase.servers.id.databases.id", "display_path": "sqldatabase.servers.id", "conditions": [ "and", - [ "sqldatabase.servers.id.databases.id.threat_detection_enabled", "false", "" ] + [ "sqldatabase.servers.id.databases.id.threat_detection.threat_detection_enabled", "false", "" ] ], - "id_suffix": "threat_detection_enabled" + "id_suffix": "db_threat_detection_disabled" } \ No newline at end of file From f5f0368112e50c697468397e1f633ad005a0681e Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 4 Mar 2019 14:22:53 -0500 Subject: [PATCH 214/667] Update rule. --- .../sqldatabase-databases-no-transparent-data-encryption.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-no-transparent-data-encryption.json b/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-no-transparent-data-encryption.json index b031259df..6955ab93b 100644 --- a/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-no-transparent-data-encryption.json +++ b/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-no-transparent-data-encryption.json @@ -1,7 +1,7 @@ { "dashboard_name": "SQL Databases", - "description": "Transparent Data Encryption disabled", - "rationale": "You should enable transparent data encryption for all of your SQL databases. See CIS 4.2.6.", + "description": "Transparent Data Encryption is disabled for some SQL databases", + "rationale": "You should enable transparent data encryption for all of your SQL databases (CIS 4.2.6).", "path": "sqldatabase.servers.id.databases.id", "display_path": "sqldatabase.servers.id", "conditions": [ "and", From 5d4e4df6031c192ab736f04107a57f6ca87bd90a Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 4 Mar 2019 14:23:09 -0500 Subject: [PATCH 215/667] Add new rule. --- ...se-databases-threat-detection-disabled-alerts.json | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-threat-detection-disabled-alerts.json diff --git a/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-threat-detection-disabled-alerts.json b/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-threat-detection-disabled-alerts.json new file mode 100644 index 000000000..3036550a4 --- /dev/null +++ b/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-threat-detection-disabled-alerts.json @@ -0,0 +1,11 @@ +{ + "dashboard_name": "SQL Databases", + "description": "Some threat detection alerts are disabled for some SQL databases", + "rationale": "You should not disable alerts related to threat detections (CIS 4.2.3).", + "path": "sqldatabase.servers.id.databases.id", + "display_path": "sqldatabase.servers.id", + "conditions": [ "and", + [ "sqldatabase.servers.id.databases.id.threat_detection.alerts_enabled", "false", "" ] + ], + "id_suffix": "db_threat_detection_alerts_disabled" +} \ No newline at end of file From 1bd76a8f5561d3b7ad342877d3432449af2cce48 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 4 Mar 2019 14:24:48 -0500 Subject: [PATCH 216/667] Add new rule. --- ...base-databases-threat-detection-low-retention.json | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-threat-detection-low-retention.json diff --git a/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-threat-detection-low-retention.json b/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-threat-detection-low-retention.json new file mode 100644 index 000000000..10b768ad8 --- /dev/null +++ b/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-threat-detection-low-retention.json @@ -0,0 +1,11 @@ +{ + "dashboard_name": "SQL Databases", + "description": "Threat detection retention period for some SQL databases is too short", + "rationale": "Threat detection retention period should be greater than _ARG_0_ days (CIS 4.2.8).", + "path": "sqldatabase.servers.id.databases.id", + "display_path": "sqldatabase.servers.id", + "conditions": [ "and", + [ "sqldatabase.servers.id.databases.id.threat_detection.retention_days", "lessThan", "_ARG_0_" ] + ], + "id_suffix": "db_low_threat_detection_retention" +} \ No newline at end of file From 84dc2e84a796e7f3a8fc905216f06fafc4b4fc16 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 4 Mar 2019 14:25:02 -0500 Subject: [PATCH 217/667] Add new rule. --- ...tabases-threat-detection-send-alerts-disabled.json | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-threat-detection-send-alerts-disabled.json diff --git a/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-threat-detection-send-alerts-disabled.json b/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-threat-detection-send-alerts-disabled.json new file mode 100644 index 000000000..f9d6f4d1b --- /dev/null +++ b/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-threat-detection-send-alerts-disabled.json @@ -0,0 +1,11 @@ +{ + "dashboard_name": "SQL Databases", + "description": "Send threat detection alerts is disabled for some SQL databases", + "rationale": "You should specify email addresses and ensure that alerts are sent to them (CIS 4.2.4 and 4.2.5).", + "path": "sqldatabase.servers.id.databases.id", + "display_path": "sqldatabase.servers.id", + "conditions": [ "and", + [ "sqldatabase.servers.id.databases.id.threat_detection.send_alerts_enabled", "false", "" ] + ], + "id_suffix": "db_send_threat_detection_alerts_disabled" +} \ No newline at end of file From d108caa7f070d256eff9113a3af391d7e4ee25b2 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 4 Mar 2019 14:25:15 -0500 Subject: [PATCH 218/667] Add new rule. --- .../sqldatabase-servers-auditing-low-retention.json | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 ScoutSuite/providers/azure/rules/findings/sqldatabase-servers-auditing-low-retention.json diff --git a/ScoutSuite/providers/azure/rules/findings/sqldatabase-servers-auditing-low-retention.json b/ScoutSuite/providers/azure/rules/findings/sqldatabase-servers-auditing-low-retention.json new file mode 100644 index 000000000..30a559f39 --- /dev/null +++ b/ScoutSuite/providers/azure/rules/findings/sqldatabase-servers-auditing-low-retention.json @@ -0,0 +1,11 @@ +{ + "dashboard_name": "SQL Servers", + "description": "Auditing retention period for some SQL servers is too short", + "rationale": "Auditing retention period should be greater than _ARG_0_ days (CIS 4.1.6).", + "path": "sqldatabase.servers.id", + "display_path": "sqldatabase.servers.id", + "conditions": [ "and", + [ "sqldatabase.servers.id.auditing.retention_days", "lessThan", "_ARG_0_" ] + ], + "id_suffix": "server_low_auditing_retention" +} \ No newline at end of file From 26db37673731d7016619ee010f89eff5b0695848 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 4 Mar 2019 14:25:29 -0500 Subject: [PATCH 219/667] Update rule. --- .../findings/sqldatabase-servers-no-ad-admin-configured.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/azure/rules/findings/sqldatabase-servers-no-ad-admin-configured.json b/ScoutSuite/providers/azure/rules/findings/sqldatabase-servers-no-ad-admin-configured.json index 78d43f37e..dad58faa2 100644 --- a/ScoutSuite/providers/azure/rules/findings/sqldatabase-servers-no-ad-admin-configured.json +++ b/ScoutSuite/providers/azure/rules/findings/sqldatabase-servers-no-ad-admin-configured.json @@ -1,7 +1,7 @@ { "dashboard_name": "SQL Databases", - "description": "Azure Active Directory Admin not configured", - "rationale": "You should set an Azure Active Directory admin for every SQL server. See CIS 4.1.8.", + "description": "Azure Active Directory Admin is not configured for some SQL servers", + "rationale": "You should set an Azure Active Directory admin for every SQL server (CIS 4.1.8).", "path": "sqldatabase.servers.id", "display_path": "sqldatabase.servers.id", "conditions": [ "and", From 9d765720021900a8faaab8f1342d8dd93da840a6 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 4 Mar 2019 14:29:27 -0500 Subject: [PATCH 220/667] Update rule. --- .../rules/findings/sqldatabase-servers-no-auditing.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ScoutSuite/providers/azure/rules/findings/sqldatabase-servers-no-auditing.json b/ScoutSuite/providers/azure/rules/findings/sqldatabase-servers-no-auditing.json index 1c8d6bcaa..4eb19be22 100644 --- a/ScoutSuite/providers/azure/rules/findings/sqldatabase-servers-no-auditing.json +++ b/ScoutSuite/providers/azure/rules/findings/sqldatabase-servers-no-auditing.json @@ -1,11 +1,11 @@ { "dashboard_name": "SQL Servers", - "description": "Auditing disabled on SQL servers", + "description": "Auditing is disabled for some SQL servers", "rationale": "You should enable auditing for all of your SQL servers (CIS 4.1.1).", "path": "sqldatabase.servers.id", "display_path": "sqldatabase.servers.id", "conditions": [ "and", - [ "sqldatabase.servers.id.auditing_enabled", "false", "" ] + [ "sqldatabase.servers.id.auditing.auditing_enabled", "false", "" ] ], - "id_suffix": "auditing_enabled" + "id_suffix": "server_auditing_disabled" } \ No newline at end of file From a95d869d849ac84c4de9032f6d597c15c2b1a66a Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 4 Mar 2019 14:40:51 -0500 Subject: [PATCH 221/667] Add new rule. --- .../sqldatabase-servers-no-threat-detection.json | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 ScoutSuite/providers/azure/rules/findings/sqldatabase-servers-no-threat-detection.json diff --git a/ScoutSuite/providers/azure/rules/findings/sqldatabase-servers-no-threat-detection.json b/ScoutSuite/providers/azure/rules/findings/sqldatabase-servers-no-threat-detection.json new file mode 100644 index 000000000..c76e13638 --- /dev/null +++ b/ScoutSuite/providers/azure/rules/findings/sqldatabase-servers-no-threat-detection.json @@ -0,0 +1,11 @@ +{ + "dashboard_name": "SQL Servers", + "description": "Threat detection is disabled for some SQL servers", + "rationale": "You should enable threat detection for all of your SQL servers (CIS 4.1.2).", + "path": "sqldatabase.servers.id", + "display_path": "sqldatabase.servers.id", + "conditions": [ "and", + [ "sqldatabase.servers.id.threat_detection.threat_detection_enabled", "false", "" ] + ], + "id_suffix": "server_threat_detection_disabled" +} \ No newline at end of file From abdf3a9c9f6a513fc9e17acf948838abdc85b60f Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 4 Mar 2019 14:41:36 -0500 Subject: [PATCH 222/667] Add new rule. --- ...base-servers-threat-detection-disabled-alerts.json | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 ScoutSuite/providers/azure/rules/findings/sqldatabase-servers-threat-detection-disabled-alerts.json diff --git a/ScoutSuite/providers/azure/rules/findings/sqldatabase-servers-threat-detection-disabled-alerts.json b/ScoutSuite/providers/azure/rules/findings/sqldatabase-servers-threat-detection-disabled-alerts.json new file mode 100644 index 000000000..af0ffe90e --- /dev/null +++ b/ScoutSuite/providers/azure/rules/findings/sqldatabase-servers-threat-detection-disabled-alerts.json @@ -0,0 +1,11 @@ +{ + "dashboard_name": "SQL Servers", + "description": "Some threat detection alerts have been disabled for some SQL servers", + "rationale": "You should not disable alerts related to threat detections (CIS 4.1.3).", + "path": "sqldatabase.servers.id", + "display_path": "sqldatabase.servers.id", + "conditions": [ "and", + [ "sqldatabase.servers.id.threat_detection.alerts_enabled", "false", "" ] + ], + "id_suffix": "server_threat_detection_alerts_disabled" +} \ No newline at end of file From e61a9b2ee6a611eb83b68cdaabc33ab10d4772b4 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 4 Mar 2019 14:42:01 -0500 Subject: [PATCH 223/667] Add new rule. --- ...tabase-servers-threat-detection-low-retention.json | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 ScoutSuite/providers/azure/rules/findings/sqldatabase-servers-threat-detection-low-retention.json diff --git a/ScoutSuite/providers/azure/rules/findings/sqldatabase-servers-threat-detection-low-retention.json b/ScoutSuite/providers/azure/rules/findings/sqldatabase-servers-threat-detection-low-retention.json new file mode 100644 index 000000000..81ea87317 --- /dev/null +++ b/ScoutSuite/providers/azure/rules/findings/sqldatabase-servers-threat-detection-low-retention.json @@ -0,0 +1,11 @@ +{ + "dashboard_name": "SQL Servers", + "description": "Threat detection retention period for some SQL servers is too short", + "rationale": "Threat detection retention period should be greater than _ARG_0_ days (CIS 4.1.7).", + "path": "sqldatabase.servers.id", + "display_path": "sqldatabase.servers.id", + "conditions": [ "and", + [ "sqldatabase.servers.id.threat_detection.retention_days", "lessThan", "_ARG_0_" ] + ], + "id_suffix": "server_low_threat_detection_retention" +} \ No newline at end of file From f5287693d0777e99463210f77a5f301c9ea3e9d8 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 4 Mar 2019 14:42:31 -0500 Subject: [PATCH 224/667] Add new rule. --- ...servers-threat-detection-send-alerts-disabled.json | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 ScoutSuite/providers/azure/rules/findings/sqldatabase-servers-threat-detection-send-alerts-disabled.json diff --git a/ScoutSuite/providers/azure/rules/findings/sqldatabase-servers-threat-detection-send-alerts-disabled.json b/ScoutSuite/providers/azure/rules/findings/sqldatabase-servers-threat-detection-send-alerts-disabled.json new file mode 100644 index 000000000..a327e5f8f --- /dev/null +++ b/ScoutSuite/providers/azure/rules/findings/sqldatabase-servers-threat-detection-send-alerts-disabled.json @@ -0,0 +1,11 @@ +{ + "dashboard_name": "SQL Servers", + "description": "Send threat detection alerts is disabled for some SQL servers", + "rationale": "You should specify email addresses and ensure that alerts are sent to them (CIS 4.1.4 and 4.1.5).", + "path": "sqldatabase.servers.id", + "display_path": "sqldatabase.servers.id", + "conditions": [ "and", + [ "sqldatabase.servers.id.threat_detection.send_alerts_enabled", "false", "" ] + ], + "id_suffix": "server_send_threat_detection_alerts_disabled" +} \ No newline at end of file From 18fbe6f42523b6f52c3c1cc3add6dc2dcf825089 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 4 Mar 2019 14:47:27 -0500 Subject: [PATCH 225/667] Add the new rules to the default ruleset. --- .../azure/rules/rulesets/default.json | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/ScoutSuite/providers/azure/rules/rulesets/default.json b/ScoutSuite/providers/azure/rules/rulesets/default.json index a3102b687..c83ccff09 100644 --- a/ScoutSuite/providers/azure/rules/rulesets/default.json +++ b/ScoutSuite/providers/azure/rules/rulesets/default.json @@ -60,6 +60,15 @@ "level": "warning" } ], + "sqldatabase-databases-auditing-low-retention.json": [ + { + "args": [ + "90" + ], + "enabled": true, + "level": "warning" + } + ], "sqldatabase-databases-no-auditing.json": [ { "enabled": true, @@ -78,6 +87,36 @@ "level": "warning" } ], + "sqldatabase-databases-threat-detection-disabled-alerts.json": [ + { + "enabled": true, + "level": "warning" + } + ], + "sqldatabase-databases-threat-detection-low-retention.json": [ + { + "args": [ + "90" + ], + "enabled": true, + "level": "warning" + } + ], + "sqldatabase-databases-threat-detection-send-alerts-disabled.json": [ + { + "enabled": true, + "level": "warning" + } + ], + "sqldatabase-servers-auditing-low-retention.json": [ + { + "args": [ + "90" + ], + "enabled": true, + "level": "warning" + } + ], "sqldatabase-servers-no-ad-admin-configured.json": [ { "enabled": true, @@ -90,6 +129,33 @@ "level": "warning" } ], + "sqldatabase-servers-no-threat-detection.json": [ + { + "enabled": true, + "level": "warning" + } + ], + "sqldatabase-servers-threat-detection-disabled-alerts.json": [ + { + "enabled": true, + "level": "warning" + } + ], + "sqldatabase-servers-threat-detection-low-retention.json": [ + { + "args": [ + "90" + ], + "enabled": true, + "level": "warning" + } + ], + "sqldatabase-servers-threat-detection-send-alerts-disabled.json": [ + { + "enabled": true, + "level": "warning" + } + ], "securitycenter-standard-tier-not-enabled.json": [ { "enabled": true, From 396cba0a74759bf9717945b8551b78fbbb409d91 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 4 Mar 2019 14:50:00 -0500 Subject: [PATCH 226/667] Update HTML template. --- .../azure/services.sqldatabase.servers.html | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/ScoutSuite/output/data/html/partials/azure/services.sqldatabase.servers.html b/ScoutSuite/output/data/html/partials/azure/services.sqldatabase.servers.html index 1f277b709..2f0491fe3 100644 --- a/ScoutSuite/output/data/html/partials/azure/services.sqldatabase.servers.html +++ b/ScoutSuite/output/data/html/partials/azure/services.sqldatabase.servers.html @@ -8,7 +8,12 @@

    {{name}}

    Information

    SQL Server Name: {{name}}
    Azure Active Directory admin configured: {{ad_admin_configured}}
    -
    Auditing: {{ convert_bool_to_enabled auditing_enabled }}
    +
    Auditing: {{ convert_bool_to_enabled auditing.auditing_enabled }}
    +
    Auditing retention period: {{ auditing.retention_days }}
    +
    Threat detection: {{ convert_bool_to_enabled threat_detection.threat_detection_enabled }}
    +
    Threat detection alerts: {{ convert_bool_to_enabled threat_detection.alerts_enabled }}
    +
    Send threat detection alerts: {{ convert_bool_to_enabled threat_detection.send_alerts_enabled }}
    +
    Threat detection retention period: {{ threat_detection.retention_days }}
    @@ -17,8 +22,12 @@

    SQL Databases

    {{#each databases}} {{@key}}
    -
    Auditing: {{ convert_bool_to_enabled auditing_enabled }}
    -
    Threat detection: {{ convert_bool_to_enabled threat_detection_enabled }}
    +
    Auditing: {{ convert_bool_to_enabled auditing.auditing_enabled }}
    +
    Auditing retention period: {{ auditing.retention_days }}
    +
    Threat detection: {{ convert_bool_to_enabled threat_detection.threat_detection_enabled }}
    +
    Threat detection alerts: {{ convert_bool_to_enabled threat_detection.alerts_enabled }}
    +
    Send threat detection alerts: {{ convert_bool_to_enabled threat_detection.send_alerts_enabled }}
    +
    Threat detection retention period: {{ threat_detection.retention_days }}
    Transparent data encryption: {{ convert_bool_to_enabled transparent_data_encryption_enabled }}
    Geo-replication configured: {{ replication_configured }}
    From 0869257f4004d5749f261ad30927e5c301915257 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sgha=C3=AFer?= Date: Mon, 4 Mar 2019 16:11:10 -0500 Subject: [PATCH 227/667] Update ScoutSuite/providers/aws/resources/cloudformation/service.py Co-Authored-By: Aboisier --- ScoutSuite/providers/aws/resources/cloudformation/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/resources/cloudformation/service.py b/ScoutSuite/providers/aws/resources/cloudformation/service.py index 150fa4a8c..b61f5614a 100644 --- a/ScoutSuite/providers/aws/resources/cloudformation/service.py +++ b/ScoutSuite/providers/aws/resources/cloudformation/service.py @@ -3,7 +3,7 @@ from ScoutSuite.providers.aws.facade.facade import AWSFacade -class Stack(AWSResources): +class Stacks(AWSResources): async def fetch_all(self, **kwargs): raw_stacks = self.facade.cloudformation.get_stacks(self.scope['region']) for raw_stack in raw_stacks: From 817bd6ac65acd6bb8eefb0d0dcc097e015a272ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sgha=C3=AFer?= Date: Mon, 4 Mar 2019 16:11:16 -0500 Subject: [PATCH 228/667] Update ScoutSuite/providers/aws/resources/cloudformation/service.py Co-Authored-By: Aboisier --- ScoutSuite/providers/aws/resources/cloudformation/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/resources/cloudformation/service.py b/ScoutSuite/providers/aws/resources/cloudformation/service.py index b61f5614a..a114ae426 100644 --- a/ScoutSuite/providers/aws/resources/cloudformation/service.py +++ b/ScoutSuite/providers/aws/resources/cloudformation/service.py @@ -28,7 +28,7 @@ def _parse_stack(self, raw_stack): class CloudFormation(Regions): _children = [ - (Stack, 'stacks') + (Stacks, 'stacks') ] def __init__(self): From 25b698bc3d7e48a0f495c217c4f452bdb7f9eda6 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 4 Mar 2019 16:32:29 -0500 Subject: [PATCH 229/667] Removed duplicated method --- ScoutSuite/providers/base/utils.py | 14 -------------- ScoutSuite/providers/utils.py | 18 +++++++++--------- 2 files changed, 9 insertions(+), 23 deletions(-) delete mode 100644 ScoutSuite/providers/base/utils.py diff --git a/ScoutSuite/providers/base/utils.py b/ScoutSuite/providers/base/utils.py deleted file mode 100644 index a944d0ae6..000000000 --- a/ScoutSuite/providers/base/utils.py +++ /dev/null @@ -1,14 +0,0 @@ -from hashlib import sha1 - - -def get_non_provider_id(name): - """ - Not all AWS resources have an ID and some services allow the use of "." in names, which break's Scout2's - recursion scheme if name is used as an ID. Use SHA1(name) instead. - - :param name: Name of the resource to - :return: SHA1(name) - """ - m = sha1() - m.update(name.encode('utf-8')) - return m.hexdigest() diff --git a/ScoutSuite/providers/utils.py b/ScoutSuite/providers/utils.py index dfbf6e514..7096ba531 100644 --- a/ScoutSuite/providers/utils.py +++ b/ScoutSuite/providers/utils.py @@ -2,13 +2,13 @@ def get_non_provider_id(name): - """ - Not all resources have an ID and some services allow the use of "." in names, which breaks Scout's - recursion scheme if name is used as an ID. Use SHA1(name) instead. + """ + Not all resources have an ID and some services allow the use of "." in names, which breaks Scout's + recursion scheme if name is used as an ID. Use SHA1(name) instead. - :param name: Name of the resource to - :return: SHA1(name) - """ - _hash = sha1() - _hash.update(name.encode('utf-8')) - return _hash.hexdigest() + :param name: Name of the resource to + :return: SHA1(name) + """ + hash = sha1() + hash.update(name.encode('utf-8')) + return hash.hexdigest() From b6aa4cdedfd5d78cf06af20105219e46d9d26d22 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 4 Mar 2019 16:33:11 -0500 Subject: [PATCH 230/667] Fixed import --- ScoutSuite/providers/aws/resources/cloudtrail/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/resources/cloudtrail/service.py b/ScoutSuite/providers/aws/resources/cloudtrail/service.py index 826d014ff..206ebd5cb 100644 --- a/ScoutSuite/providers/aws/resources/cloudtrail/service.py +++ b/ScoutSuite/providers/aws/resources/cloudtrail/service.py @@ -1,7 +1,7 @@ from ScoutSuite.providers.aws.resources.regions import Regions from ScoutSuite.providers.aws.resources.resources import AWSResources from ScoutSuite.providers.aws.facade.facade import AWSFacade -from ScoutSuite.providers.base.utils import get_non_provider_id +from ScoutSuite.providers.utils import get_non_provider_id import time From a566726eadf87312b7038b2d7ca1b05cb19db7ae Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 4 Mar 2019 16:53:17 -0500 Subject: [PATCH 231/667] Created cloudwatch service file --- .../aws/resources/cloudwatch/service.py | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 ScoutSuite/providers/aws/resources/cloudwatch/service.py diff --git a/ScoutSuite/providers/aws/resources/cloudwatch/service.py b/ScoutSuite/providers/aws/resources/cloudwatch/service.py new file mode 100644 index 000000000..5204c6022 --- /dev/null +++ b/ScoutSuite/providers/aws/resources/cloudwatch/service.py @@ -0,0 +1,49 @@ +# # -*- coding: utf-8 -*- +# """ +# CloudWatch-related classes and functions +# """ + +# from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig + + +# ######################################## +# # CloudWatchRegionConfig +# ######################################## + +# class CloudWatchRegionConfig(RegionConfig): +# """ +# CloudWatch configuration for a single AWS region +# """ +# alarms = {} + +# def parse_alarm(self, global_params, region, alarm): +# """ +# Parse a single CloudWatch trail + +# :param global_params: Parameters shared for all regions +# :param region: Name of the AWS region +# :param alarm: Alarm +# """ +# alarm['arn'] = alarm.pop('AlarmArn') +# alarm['name'] = alarm.pop('AlarmName') +# # Drop some data +# for k in ['AlarmConfigurationUpdatedTimestamp', 'StateReason', 'StateReasonData', 'StateUpdatedTimestamp']: +# if k in alarm: +# alarm.pop(k) +# alarm_id = self.get_non_provider_id(alarm['arn']) +# self.alarms[alarm_id] = alarm + + +# ######################################## +# # CloudWatchConfig +# ######################################## + +# class CloudWatchConfig(RegionalServiceConfig): +# """ +# CloudWatch configuration for all AWS regions +# """ + +# region_config_class = CloudWatchRegionConfig + +# def __init__(self, service_metadata, thread_config=4): +# super(CloudWatchConfig, self).__init__(service_metadata, thread_config) From c957de5e045002be579b8ea3c8aac21dcdce0e2e Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 4 Mar 2019 17:52:45 -0500 Subject: [PATCH 232/667] Reordered imports --- ScoutSuite/providers/aws/resources/awslambda/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/resources/awslambda/service.py b/ScoutSuite/providers/aws/resources/awslambda/service.py index 7ec5fbb35..64311438c 100644 --- a/ScoutSuite/providers/aws/resources/awslambda/service.py +++ b/ScoutSuite/providers/aws/resources/awslambda/service.py @@ -1,6 +1,6 @@ +from ScoutSuite.providers.aws.facade.facade import AWSFacade from ScoutSuite.providers.aws.resources.regions import Regions from ScoutSuite.providers.aws.resources.resources import AWSResources -from ScoutSuite.providers.aws.facade.facade import AWSFacade class RegionalLambdas(AWSResources): From 9b5d9a61bdf66831ee984986177ce09b3de7b85f Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 4 Mar 2019 17:53:05 -0500 Subject: [PATCH 233/667] Reordered imports --- ScoutSuite/providers/aws/facade/facade.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ScoutSuite/providers/aws/facade/facade.py b/ScoutSuite/providers/aws/facade/facade.py index baf02896d..e57802141 100644 --- a/ScoutSuite/providers/aws/facade/facade.py +++ b/ScoutSuite/providers/aws/facade/facade.py @@ -1,8 +1,13 @@ from botocore.session import Session from collections import Counter +from botocore.session import Session + +from ScoutSuite.providers.aws.facade.awslambda import LambdaFacade +from ScoutSuite.providers.aws.facade.cloudwatch import CloudWatch from ScoutSuite.providers.aws.facade.ec2 import EC2Facade from ScoutSuite.providers.aws.facade.awslambda import LambdaFacade + class AWSFacade(object): def __init__(self): self.ec2 = EC2Facade() From 602d233747ccf1270cf7bc15aa4ca9ad68c62f7a Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 4 Mar 2019 17:53:20 -0500 Subject: [PATCH 234/667] Added line at the end of file for @misg --- ScoutSuite/providers/aws/facade/facade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/facade/facade.py b/ScoutSuite/providers/aws/facade/facade.py index e57802141..96180ed5a 100644 --- a/ScoutSuite/providers/aws/facade/facade.py +++ b/ScoutSuite/providers/aws/facade/facade.py @@ -26,4 +26,4 @@ async def build_region_list(self, service: str, chosen_regions=None, partition_n if chosen_regions: return list((Counter(regions) & Counter(chosen_regions)).elements()) else: - return regions \ No newline at end of file + return regions From e5da70dc21ad07cc7f799c42262560d15353c44a Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 4 Mar 2019 17:54:08 -0500 Subject: [PATCH 235/667] Implemented cloudwatch facade --- ScoutSuite/providers/aws/facade/cloudwatch.py | 6 ++++++ ScoutSuite/providers/aws/facade/facade.py | 4 +--- 2 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 ScoutSuite/providers/aws/facade/cloudwatch.py diff --git a/ScoutSuite/providers/aws/facade/cloudwatch.py b/ScoutSuite/providers/aws/facade/cloudwatch.py new file mode 100644 index 000000000..07a58e405 --- /dev/null +++ b/ScoutSuite/providers/aws/facade/cloudwatch.py @@ -0,0 +1,6 @@ +from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils + + +class CloudWatch: + def get_alarms(self, region): + return AWSFacadeUtils.get_all_pages('cloudwatch', region, 'describe_alarms', 'MetricAlarms') diff --git a/ScoutSuite/providers/aws/facade/facade.py b/ScoutSuite/providers/aws/facade/facade.py index 96180ed5a..300e18f4d 100644 --- a/ScoutSuite/providers/aws/facade/facade.py +++ b/ScoutSuite/providers/aws/facade/facade.py @@ -1,18 +1,16 @@ -from botocore.session import Session from collections import Counter from botocore.session import Session from ScoutSuite.providers.aws.facade.awslambda import LambdaFacade from ScoutSuite.providers.aws.facade.cloudwatch import CloudWatch from ScoutSuite.providers.aws.facade.ec2 import EC2Facade -from ScoutSuite.providers.aws.facade.awslambda import LambdaFacade class AWSFacade(object): def __init__(self): self.ec2 = EC2Facade() self.awslambda = LambdaFacade() - + self.cloudwatch = CloudWatch() async def build_region_list(self, service: str, chosen_regions=None, partition_name='aws'): service = 'ec2containerservice' if service == 'ecs' else service From 55b7fffc941c5676578a7ec92aa9500ba2dd9d62 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 4 Mar 2019 17:54:29 -0500 Subject: [PATCH 236/667] Migrated CloudWatch to the new architecture --- .../aws/resources/cloudwatch/service.py | 91 +++++++++---------- 1 file changed, 42 insertions(+), 49 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/cloudwatch/service.py b/ScoutSuite/providers/aws/resources/cloudwatch/service.py index 5204c6022..90a9ee022 100644 --- a/ScoutSuite/providers/aws/resources/cloudwatch/service.py +++ b/ScoutSuite/providers/aws/resources/cloudwatch/service.py @@ -1,49 +1,42 @@ -# # -*- coding: utf-8 -*- -# """ -# CloudWatch-related classes and functions -# """ - -# from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig - - -# ######################################## -# # CloudWatchRegionConfig -# ######################################## - -# class CloudWatchRegionConfig(RegionConfig): -# """ -# CloudWatch configuration for a single AWS region -# """ -# alarms = {} - -# def parse_alarm(self, global_params, region, alarm): -# """ -# Parse a single CloudWatch trail - -# :param global_params: Parameters shared for all regions -# :param region: Name of the AWS region -# :param alarm: Alarm -# """ -# alarm['arn'] = alarm.pop('AlarmArn') -# alarm['name'] = alarm.pop('AlarmName') -# # Drop some data -# for k in ['AlarmConfigurationUpdatedTimestamp', 'StateReason', 'StateReasonData', 'StateUpdatedTimestamp']: -# if k in alarm: -# alarm.pop(k) -# alarm_id = self.get_non_provider_id(alarm['arn']) -# self.alarms[alarm_id] = alarm - - -# ######################################## -# # CloudWatchConfig -# ######################################## - -# class CloudWatchConfig(RegionalServiceConfig): -# """ -# CloudWatch configuration for all AWS regions -# """ - -# region_config_class = CloudWatchRegionConfig - -# def __init__(self, service_metadata, thread_config=4): -# super(CloudWatchConfig, self).__init__(service_metadata, thread_config) +from ScoutSuite.providers.aws.facade.facade import AWSFacade +from ScoutSuite.providers.aws.resources.regions import Regions +from ScoutSuite.providers.aws.resources.resources import AWSResources + +from ScoutSuite.providers.utils import get_non_provider_id + + +class Alarms(AWSResources): + async def fetch_all(self, **kwargs): + raw_alarms = self.facade.cloudwatch.get_alarms(self.scope['region']) + for raw_alarm in raw_alarms: + name, resource = self._parse(raw_alarm) + self[name] = resource + + def _parse(self, raw_alarm): + """ + Parse a single CloudWatch trail + + :param global_params: Parameters shared for all regions + :param region: Name of the AWS region + :param alarm: Alarm + """ + + raw_alarm['arn'] = raw_alarm.pop('AlarmArn') + raw_alarm['name'] = raw_alarm.pop('AlarmName') + + # Drop some data + for k in ['AlarmConfigurationUpdatedTimestamp', 'StateReason', 'StateReasonData', 'StateUpdatedTimestamp']: + if k in raw_alarm: + raw_alarm.pop(k) + + alarm_id = get_non_provider_id(raw_alarm['arn']) + return alarm_id, raw_alarm + + +class CloudWatch(Regions): + _children = [ + (Alarms, 'alarms') + ] + + def __init__(self): + super(CloudWatch, self).__init__('cloudwatch') From 7eb991ad20953b5c13944dd92bbfb9bb209818b1 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 4 Mar 2019 17:54:42 -0500 Subject: [PATCH 237/667] Integrated new CloudWatch implementation --- ScoutSuite/providers/aws/configs/services.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index 380666fa2..752f3e387 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -3,7 +3,7 @@ from ScoutSuite.providers.aws.resources.awslambda.service import Lambdas from ScoutSuite.providers.aws.services.cloudformation import CloudFormationConfig from ScoutSuite.providers.aws.services.cloudtrail import CloudTrailConfig -from ScoutSuite.providers.aws.services.cloudwatch import CloudWatchConfig +from ScoutSuite.providers.aws.resources.cloudwatch.service import CloudWatch from ScoutSuite.providers.aws.services.directconnect import DirectConnectConfig from ScoutSuite.providers.aws.resources.ec2.service import EC2 from ScoutSuite.providers.aws.services.efs import EFSConfig @@ -37,7 +37,7 @@ class AWSServicesConfig(BaseServicesConfig): Object that holds the necessary AWS configuration for all services in scope. :ivar cloudtrail: CloudTrail configuration - :ivar cloudwatch: CloudWatch configuration: TODO + :ivar cloudwatch: CloudWatch configuration: :ivar config: Config configuration :ivar dynamodb: DynomaDB configuration :ivar ec2: EC2 configuration @@ -56,7 +56,7 @@ def __init__(self, metadata=None, thread_config=4, **kwargs): super().__init__(metadata, thread_config) self.cloudformation = CloudFormationConfig(metadata['management']['cloudformation'], thread_config) self.cloudtrail = CloudTrailConfig(metadata['management']['cloudtrail'], thread_config) - self.cloudwatch = CloudWatchConfig(metadata['management']['cloudwatch'], thread_config) + self.cloudwatch = CloudWatch() self.directconnect = DirectConnectConfig(metadata['network']['directconnect'], thread_config) self.ec2 = EC2() self.efs = EFSConfig(metadata['storage']['efs'], thread_config) From 988fa92fe7a247531e643cb2a3b888b8fd2cb862 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 4 Mar 2019 17:55:12 -0500 Subject: [PATCH 238/667] Got rid of old CloudWatch implementation --- .../providers/aws/services/cloudwatch.py | 49 ------------------- 1 file changed, 49 deletions(-) delete mode 100644 ScoutSuite/providers/aws/services/cloudwatch.py diff --git a/ScoutSuite/providers/aws/services/cloudwatch.py b/ScoutSuite/providers/aws/services/cloudwatch.py deleted file mode 100644 index af3b528d8..000000000 --- a/ScoutSuite/providers/aws/services/cloudwatch.py +++ /dev/null @@ -1,49 +0,0 @@ -# -*- coding: utf-8 -*- -""" -CloudWatch-related classes and functions -""" - -from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig - - -######################################## -# CloudWatchRegionConfig -######################################## - -class CloudWatchRegionConfig(RegionConfig): - """ - CloudWatch configuration for a single AWS region - """ - alarms = {} - - def parse_alarm(self, global_params, region, alarm): - """ - Parse a single CloudWatch trail - - :param global_params: Parameters shared for all regions - :param region: Name of the AWS region - :param alarm: Alarm - """ - alarm['arn'] = alarm.pop('AlarmArn') - alarm['name'] = alarm.pop('AlarmName') - # Drop some data - for k in ['AlarmConfigurationUpdatedTimestamp', 'StateReason', 'StateReasonData', 'StateUpdatedTimestamp']: - if k in alarm: - alarm.pop(k) - alarm_id = self.get_non_provider_id(alarm['arn']) - self.alarms[alarm_id] = alarm - - -######################################## -# CloudWatchConfig -######################################## - -class CloudWatchConfig(RegionalServiceConfig): - """ - CloudWatch configuration for all AWS regions - """ - - region_config_class = CloudWatchRegionConfig - - def __init__(self, service_metadata, thread_config=4): - super(CloudWatchConfig, self).__init__(service_metadata, thread_config) From 45a52762ec39cf2b9667704a77f9706f7cc0a088 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 4 Mar 2019 17:58:43 -0500 Subject: [PATCH 239/667] Made variable more verbrose --- ScoutSuite/providers/aws/resources/cloudwatch/service.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/cloudwatch/service.py b/ScoutSuite/providers/aws/resources/cloudwatch/service.py index 90a9ee022..4b3062393 100644 --- a/ScoutSuite/providers/aws/resources/cloudwatch/service.py +++ b/ScoutSuite/providers/aws/resources/cloudwatch/service.py @@ -23,11 +23,11 @@ def _parse(self, raw_alarm): raw_alarm['arn'] = raw_alarm.pop('AlarmArn') raw_alarm['name'] = raw_alarm.pop('AlarmName') - + # Drop some data - for k in ['AlarmConfigurationUpdatedTimestamp', 'StateReason', 'StateReasonData', 'StateUpdatedTimestamp']: - if k in raw_alarm: - raw_alarm.pop(k) + for key in ['AlarmConfigurationUpdatedTimestamp', 'StateReason', 'StateReasonData', 'StateUpdatedTimestamp']: + if key in raw_alarm: + raw_alarm.pop(key) alarm_id = get_non_provider_id(raw_alarm['arn']) return alarm_id, raw_alarm From 54e365ca206a4217f3ca092bc174dea633d81b31 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 4 Mar 2019 18:02:37 -0500 Subject: [PATCH 240/667] Used a better name --- ScoutSuite/providers/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ScoutSuite/providers/utils.py b/ScoutSuite/providers/utils.py index 7096ba531..30759aa82 100644 --- a/ScoutSuite/providers/utils.py +++ b/ScoutSuite/providers/utils.py @@ -9,6 +9,6 @@ def get_non_provider_id(name): :param name: Name of the resource to :return: SHA1(name) """ - hash = sha1() - hash.update(name.encode('utf-8')) - return hash.hexdigest() + name_hash = sha1() + name_hash.update(name.encode('utf-8')) + return name_hash.hexdigest() From e7e495777de38f4e870f2dcce25080cdec8e93b0 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 4 Mar 2019 20:04:31 -0500 Subject: [PATCH 241/667] Update tests/test-aws_resources.py --- tests/test-aws_resources.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test-aws_resources.py b/tests/test-aws_resources.py index 1c85e156b..252aa14b0 100644 --- a/tests/test-aws_resources.py +++ b/tests/test-aws_resources.py @@ -31,7 +31,7 @@ async def fetch_all(self, **kwargs): class TestAWSResources(TestCase): test_dir = os.path.dirname(os.path.realpath(__file__)) - def test_lel(self): + def test_aws_composite_resource(self): loop = asyncio.new_event_loop() composite = DummyComposite({'region': 'some_region'}) loop.run_until_complete(composite.fetch_all()) @@ -43,4 +43,4 @@ def test_lel(self): actual_json = json.dumps(composite) assert (expected_json == actual_json) - \ No newline at end of file + From c57727813d8fbb0a4b33573c0f060a8755b39170 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 4 Mar 2019 20:54:47 -0500 Subject: [PATCH 242/667] Fixed indent --- tests/test-aws_resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test-aws_resources.py b/tests/test-aws_resources.py index 252aa14b0..48d5ec194 100644 --- a/tests/test-aws_resources.py +++ b/tests/test-aws_resources.py @@ -31,7 +31,7 @@ async def fetch_all(self, **kwargs): class TestAWSResources(TestCase): test_dir = os.path.dirname(os.path.realpath(__file__)) - def test_aws_composite_resource(self): + def test_aws_composite_resource(self): loop = asyncio.new_event_loop() composite = DummyComposite({'region': 'some_region'}) loop.run_until_complete(composite.fetch_all()) From b8c706a2961d24516f7a33ac8ce51c5a63f19f29 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 4 Mar 2019 21:16:13 -0500 Subject: [PATCH 243/667] Renamed method --- ScoutSuite/providers/aws/resources/cloudtrail/service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/cloudtrail/service.py b/ScoutSuite/providers/aws/resources/cloudtrail/service.py index 206ebd5cb..6b3755391 100644 --- a/ScoutSuite/providers/aws/resources/cloudtrail/service.py +++ b/ScoutSuite/providers/aws/resources/cloudtrail/service.py @@ -9,10 +9,10 @@ class Trails(AWSResources): async def fetch_all(self, **kwargs): raw_trails = self.facade.cloudtrail.get_trails(self.scope['region']) for raw_trail in raw_trails: - name, resource = self._parse_trail(raw_trail) + name, resource = self._parse(raw_trail) self[name] = resource - def _parse_trail(self, raw_trail): + def _parse(self, raw_trail): trail = {'name': raw_trail.pop('Name')} trail_id = get_non_provider_id(trail['name']) From deb0f854b743322d2c29e1abce75093fe2bab9dc Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 4 Mar 2019 21:38:46 -0500 Subject: [PATCH 244/667] Implemented DirectConnect facade --- ScoutSuite/providers/aws/facade/directconnect.py | 6 ++++++ ScoutSuite/providers/aws/facade/facade.py | 6 ++++-- 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 ScoutSuite/providers/aws/facade/directconnect.py diff --git a/ScoutSuite/providers/aws/facade/directconnect.py b/ScoutSuite/providers/aws/facade/directconnect.py new file mode 100644 index 000000000..c238644cc --- /dev/null +++ b/ScoutSuite/providers/aws/facade/directconnect.py @@ -0,0 +1,6 @@ +from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils + + +class DirectConnectFacade: + def get_connections(self, region): + return AWSFacadeUtils.get_client('directconnect', region).describe_connections()['connections'] \ No newline at end of file diff --git a/ScoutSuite/providers/aws/facade/facade.py b/ScoutSuite/providers/aws/facade/facade.py index ae29dd9bc..926219223 100644 --- a/ScoutSuite/providers/aws/facade/facade.py +++ b/ScoutSuite/providers/aws/facade/facade.py @@ -3,13 +3,15 @@ from ScoutSuite.providers.aws.facade.ec2 import EC2Facade from ScoutSuite.providers.aws.facade.awslambda import LambdaFacade from ScoutSuite.providers.aws.facade.cloudtrail import CloudTrailFacade +from ScoutSuite.providers.aws.facade.directconnect import DirectConnectFacade + class AWSFacade(object): def __init__(self): self.ec2 = EC2Facade() self.awslambda = LambdaFacade() self.cloudtrail = CloudTrailFacade() - + self.directconnect = DirectConnectFacade() async def build_region_list(self, service: str, chosen_regions=None, partition_name='aws'): service = 'ec2containerservice' if service == 'ecs' else service @@ -23,4 +25,4 @@ async def build_region_list(self, service: str, chosen_regions=None, partition_n if chosen_regions: return list((Counter(regions) & Counter(chosen_regions)).elements()) else: - return regions \ No newline at end of file + return regions From 2bc297b81b58800bce34e8c13250fbc70ba66012 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 4 Mar 2019 21:38:53 -0500 Subject: [PATCH 245/667] Migrated service to new architecture --- ScoutSuite/providers/aws/configs/services.py | 3 ++- .../aws/resources/directconnect/service.py | 25 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 ScoutSuite/providers/aws/resources/directconnect/service.py diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index 19ea0954b..392ffb5b4 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -5,6 +5,7 @@ from ScoutSuite.providers.aws.resources.cloudtrail.service import CloudTrail from ScoutSuite.providers.aws.services.cloudwatch import CloudWatchConfig from ScoutSuite.providers.aws.services.directconnect import DirectConnectConfig +from ScoutSuite.providers.aws.resources.directconnect.service import DirectConnect from ScoutSuite.providers.aws.resources.ec2.service import EC2 from ScoutSuite.providers.aws.services.efs import EFSConfig from ScoutSuite.providers.aws.services.elasticache import ElastiCacheConfig @@ -57,7 +58,7 @@ def __init__(self, metadata=None, thread_config=4, **kwargs): self.cloudformation = CloudFormationConfig(metadata['management']['cloudformation'], thread_config) self.cloudtrail = CloudTrail() self.cloudwatch = CloudWatchConfig(metadata['management']['cloudwatch'], thread_config) - self.directconnect = DirectConnectConfig(metadata['network']['directconnect'], thread_config) + self.directconnect = DirectConnect() self.ec2 = EC2() self.efs = EFSConfig(metadata['storage']['efs'], thread_config) self.elasticache = ElastiCacheConfig(metadata['database']['elasticache'], thread_config) diff --git a/ScoutSuite/providers/aws/resources/directconnect/service.py b/ScoutSuite/providers/aws/resources/directconnect/service.py new file mode 100644 index 000000000..986dd9f62 --- /dev/null +++ b/ScoutSuite/providers/aws/resources/directconnect/service.py @@ -0,0 +1,25 @@ +from ScoutSuite.providers.aws.facade.facade import AWSFacade +from ScoutSuite.providers.aws.resources.regions import Regions +from ScoutSuite.providers.aws.resources.resources import AWSResources + + +class Connections(AWSResources): + async def fetch_all(self, **kwargs): + raw_connections = self.facade.directconnect.get_connections(self.scope['region']) + for raw_connection in raw_connections: + name, resource = self._parse_function(raw_connection) + self[name] = resource + + def _parse_function(self, raw_connection): + raw_connection['id'] = raw_connection.pop('connectionId') + raw_connection['name'] = raw_connection.pop('connectionName') + return raw_connection['id'], raw_connection + + +class DirectConnect(Regions): + _children = [ + (Connections, 'connections') + ] + + def __init__(self): + super(DirectConnect, self).__init__('directconnect') From e8b0a99a220a63d5ca96c18eeb078ae8345562ac Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 5 Mar 2019 08:57:07 -0500 Subject: [PATCH 246/667] Deleted previous implementation --- .../providers/aws/services/directconnect.py | 41 ------------------- 1 file changed, 41 deletions(-) delete mode 100644 ScoutSuite/providers/aws/services/directconnect.py diff --git a/ScoutSuite/providers/aws/services/directconnect.py b/ScoutSuite/providers/aws/services/directconnect.py deleted file mode 100644 index d9b46ac93..000000000 --- a/ScoutSuite/providers/aws/services/directconnect.py +++ /dev/null @@ -1,41 +0,0 @@ -# -*- coding: utf-8 -*- - -from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig - - -######################################## -# DirectConnectRegionConfig -######################################## - -class DirectConnectRegionConfig(RegionConfig): - """ - DirectConnect configuration for a single AWS region - """ - connections = {} - - def parse_connection(self, global_params, region, connection): - """ - Parse a single connection and fetch additional attributes - - :param connection: - :param global_params: Parameters shared for all regions - :param region: Name of the AWS region - """ - connection['id'] = connection.pop('connectionId') - connection['name'] = connection.pop('connectionName') - self.connections[connection['id']] = connection - - -######################################## -# DirectConnectConfig -######################################## - -class DirectConnectConfig(RegionalServiceConfig): - """ - DirectConnect configuration for all AWS regions - """ - - region_config_class = DirectConnectRegionConfig - - def __init__(self, service_metadata, thread_config=4): - super(DirectConnectConfig, self).__init__(service_metadata, thread_config) From 0e7e5ce8a26e121aa8ece28f9306018582220e6d Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 5 Mar 2019 09:06:06 -0500 Subject: [PATCH 247/667] Removed old import --- ScoutSuite/providers/aws/configs/services.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index 392ffb5b4..a6bd00809 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -4,7 +4,6 @@ from ScoutSuite.providers.aws.services.cloudformation import CloudFormationConfig from ScoutSuite.providers.aws.resources.cloudtrail.service import CloudTrail from ScoutSuite.providers.aws.services.cloudwatch import CloudWatchConfig -from ScoutSuite.providers.aws.services.directconnect import DirectConnectConfig from ScoutSuite.providers.aws.resources.directconnect.service import DirectConnect from ScoutSuite.providers.aws.resources.ec2.service import EC2 from ScoutSuite.providers.aws.services.efs import EFSConfig From eadd82846f59dbb0d07b454fc34ff95c81fca652 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sgha=C3=AFer?= Date: Tue, 5 Mar 2019 15:27:05 -0500 Subject: [PATCH 248/667] Update description rule --- .../rules/findings/sqldatabase-databases-no-auditing.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-no-auditing.json b/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-no-auditing.json index dcfaf13b3..7c4db404c 100644 --- a/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-no-auditing.json +++ b/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-no-auditing.json @@ -1,6 +1,6 @@ { "dashboard_name": "SQL Databases", - "description": "Auditing is disabled for some SQL databases", + "description": "Auditing disabled for SQL databases", "rationale": "You should enable auditing for all of your SQL databases (CIS 4.2.1).", "path": "sqldatabase.servers.id.databases.id", "display_path": "sqldatabase.servers.id", @@ -8,4 +8,4 @@ [ "sqldatabase.servers.id.databases.id.auditing.auditing_enabled", "false", "" ] ], "id_suffix": "db_auditing_disabled" -} \ No newline at end of file +} From e0c7d4892fe924db011822863408efc314497d02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sgha=C3=AFer?= Date: Tue, 5 Mar 2019 15:27:53 -0500 Subject: [PATCH 249/667] Update description rule --- .../findings/sqldatabase-databases-no-threat-detection.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-no-threat-detection.json b/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-no-threat-detection.json index 7da59a7f7..622e2baf4 100644 --- a/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-no-threat-detection.json +++ b/ScoutSuite/providers/azure/rules/findings/sqldatabase-databases-no-threat-detection.json @@ -1,6 +1,6 @@ { "dashboard_name": "SQL Databases", - "description": "Threat detection is disabled for some SQL databases", + "description": "Theat detection disabled for SQL databases", "rationale": "You should enable threat detection for all of your SQL databases (CIS 4.2.2).", "path": "sqldatabase.servers.id.databases.id", "display_path": "sqldatabase.servers.id", @@ -8,4 +8,4 @@ [ "sqldatabase.servers.id.databases.id.threat_detection.threat_detection_enabled", "false", "" ] ], "id_suffix": "db_threat_detection_disabled" -} \ No newline at end of file +} From bd0cecb1db3edeb85238ea87bbe930a304e1f6c7 Mon Sep 17 00:00:00 2001 From: misg Date: Tue, 5 Mar 2019 16:09:43 -0500 Subject: [PATCH 250/667] Add helper to run functions concurrently. --- ScoutSuite/providers/utils.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ScoutSuite/providers/utils.py b/ScoutSuite/providers/utils.py index dfbf6e514..f1b85c030 100644 --- a/ScoutSuite/providers/utils.py +++ b/ScoutSuite/providers/utils.py @@ -1,4 +1,5 @@ from hashlib import sha1 +import asyncio def get_non_provider_id(name): @@ -12,3 +13,7 @@ def get_non_provider_id(name): _hash = sha1() _hash.update(name.encode('utf-8')) return _hash.hexdigest() + + +def run_concurrently(func): + return asyncio.get_event_loop().run_in_executor(executor=None, func=func) From 25967e57b29d9c66af0daf339162aa5e0492e3ea Mon Sep 17 00:00:00 2001 From: misg Date: Tue, 5 Mar 2019 16:11:14 -0500 Subject: [PATCH 251/667] Make AWS facade async. --- ScoutSuite/providers/aws/facade/awslambda.py | 4 +- ScoutSuite/providers/aws/facade/ec2.py | 51 +++++++++++--------- ScoutSuite/providers/aws/facade/facade.py | 10 ++-- ScoutSuite/providers/aws/facade/utils.py | 22 +++++---- 4 files changed, 48 insertions(+), 39 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/awslambda.py b/ScoutSuite/providers/aws/facade/awslambda.py index 04c3649d3..ed5d51262 100644 --- a/ScoutSuite/providers/aws/facade/awslambda.py +++ b/ScoutSuite/providers/aws/facade/awslambda.py @@ -2,5 +2,5 @@ class LambdaFacade: - def get_functions(self, region): - return AWSFacadeUtils.get_all_pages('lambda', region, 'list_functions', 'Functions') + async def get_functions(self, region): + return await AWSFacadeUtils.get_all_pages('lambda', region, 'list_functions', 'Functions') diff --git a/ScoutSuite/providers/aws/facade/ec2.py b/ScoutSuite/providers/aws/facade/ec2.py index a884a1a70..febb1da69 100644 --- a/ScoutSuite/providers/aws/facade/ec2.py +++ b/ScoutSuite/providers/aws/facade/ec2.py @@ -1,23 +1,25 @@ import boto3 import base64 -import itertools from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils +from ScoutSuite.providers.utils import run_concurrently class EC2Facade: - def get_instance_user_data(self, region: str, instance_id: str): - ec2_client = AWSFacadeUtils.get_client('ec2', region) - user_data_response = ec2_client.describe_instance_attribute(Attribute='userData', InstanceId=instance_id) + async def get_instance_user_data(self, region: str, instance_id: str): + ec2_client = await AWSFacadeUtils.get_client('ec2', region) + user_data_response = await run_concurrently( + lambda: ec2_client.describe_instance_attribute(Attribute='userData', InstanceId=instance_id)) if 'Value' not in user_data_response['UserData'].keys(): return None return base64.b64decode(user_data_response['UserData']['Value']).decode('utf-8') - def get_instances(self, region, vpc): + async def get_instances(self, region, vpc): filters = [{'Name': 'vpc-id', 'Values': [vpc]}] - reservations = AWSFacadeUtils.get_all_pages('ec2', region, 'describe_instances', 'Reservations', Filters=filters) + reservations =\ + await AWSFacadeUtils.get_all_pages('ec2', region, 'describe_instances', 'Reservations', Filters=filters) instances = [] for reservation in reservations: @@ -27,36 +29,39 @@ def get_instances(self, region, vpc): return instances - def get_security_groups(self, region, vpc): + async def get_security_groups(self, region, vpc): filters = [{'Name': 'vpc-id', 'Values': [vpc]}] - return AWSFacadeUtils.get_all_pages('ec2', region, 'describe_security_groups', 'SecurityGroups', Filters=filters) + return await AWSFacadeUtils.get_all_pages( + 'ec2', region, 'describe_security_groups', 'SecurityGroups', Filters=filters) - def get_vpcs(self, region): - ec2_client = boto3.client('ec2', region_name=region) - return ec2_client.describe_vpcs()['Vpcs'] + async def get_vpcs(self, region): + ec2_client = await run_concurrently(lambda: boto3.client('ec2', region_name=region)) + return await run_concurrently(lambda: ec2_client.describe_vpcs()['Vpcs']) - def get_images(self, region, owner_id): + async def get_images(self, region, owner_id): filters = [{'Name': 'owner-id', 'Values': [owner_id]}] - response = AWSFacadeUtils.get_client('ec2', region) \ - .describe_images(Filters=filters) + client = await AWSFacadeUtils.get_client('ec2', region) + response = await run_concurrently(lambda: client.describe_images(Filters=filters)) return response['Images'] - def get_network_interfaces(self, region, vpc): + async def get_network_interfaces(self, region, vpc): filters = [{'Name': 'vpc-id', 'Values': [vpc]}] - return AWSFacadeUtils.get_all_pages('ec2', region, 'describe_network_interfaces', 'NetworkInterfaces', Filters=filters) + return await AWSFacadeUtils.get_all_pages( + 'ec2', region, 'describe_network_interfaces', 'NetworkInterfaces', Filters=filters) - def get_volumes(self, region): - return AWSFacadeUtils.get_all_pages('ec2', region, 'describe_volumes', 'Volumes') + async def get_volumes(self, region): + return await AWSFacadeUtils.get_all_pages('ec2', region, 'describe_volumes', 'Volumes') - def get_snapshots(self, region, owner_id): + async def get_snapshots(self, region, owner_id): filters = [{'Name': 'owner-id', 'Values': [owner_id]}] - snapshots = AWSFacadeUtils.get_all_pages('ec2', region, 'describe_snapshots', 'Snapshots', Filters=filters) + snapshots = await AWSFacadeUtils.get_all_pages( + 'ec2', region, 'describe_snapshots', 'Snapshots', Filters=filters) - ec2_client = AWSFacadeUtils.get_client('ec2', region) + ec2_client = await AWSFacadeUtils.get_client('ec2', region) for snapshot in snapshots: - snapshot['CreateVolumePermissions'] = ec2_client.describe_snapshot_attribute( + snapshot['CreateVolumePermissions'] = await run_concurrently(lambda: ec2_client.describe_snapshot_attribute( Attribute='createVolumePermission', - SnapshotId=snapshot['SnapshotId'])['CreateVolumePermissions'] + SnapshotId=snapshot['SnapshotId'])['CreateVolumePermissions']) return snapshots diff --git a/ScoutSuite/providers/aws/facade/facade.py b/ScoutSuite/providers/aws/facade/facade.py index baf02896d..9e87f039b 100644 --- a/ScoutSuite/providers/aws/facade/facade.py +++ b/ScoutSuite/providers/aws/facade/facade.py @@ -2,23 +2,25 @@ from collections import Counter from ScoutSuite.providers.aws.facade.ec2 import EC2Facade from ScoutSuite.providers.aws.facade.awslambda import LambdaFacade +from ScoutSuite.providers.utils import run_concurrently + class AWSFacade(object): + def __init__(self): self.ec2 = EC2Facade() self.awslambda = LambdaFacade() - async def build_region_list(self, service: str, chosen_regions=None, partition_name='aws'): service = 'ec2containerservice' if service == 'ecs' else service - available_services = Session().get_available_services() + available_services = await run_concurrently(lambda: Session().get_available_services()) if service not in available_services: raise Exception('Service ' + service + ' is not available.') - regions = Session().get_available_regions(service, partition_name) + regions = await run_concurrently(lambda: Session().get_available_regions(service, partition_name)) if chosen_regions: return list((Counter(regions) & Counter(chosen_regions)).elements()) else: - return regions \ No newline at end of file + return regions diff --git a/ScoutSuite/providers/aws/facade/utils.py b/ScoutSuite/providers/aws/facade/utils.py index 85ffb2bc0..da273f72e 100644 --- a/ScoutSuite/providers/aws/facade/utils.py +++ b/ScoutSuite/providers/aws/facade/utils.py @@ -1,20 +1,21 @@ -from typing import Callable import boto3 +from ScoutSuite.providers.utils import run_concurrently + + # TODO: Add docstrings class AWSFacadeUtils: _clients = {} - + @staticmethod - def get_all_pages(service: str, region: str, paginator_name: str, response_key: str, **paginator_args): - pages = AWSFacadeUtils.get_client(service, region) \ - .get_paginator(paginator_name) \ - .paginate(**paginator_args) - + async def get_all_pages(service: str, region: str, paginator_name: str, response_key: str, **paginator_args): + client = await AWSFacadeUtils.get_client(service, region) + pages = await run_concurrently(lambda: client.get_paginator(paginator_name).paginate(**paginator_args)) + return AWSFacadeUtils._get_from_all_pages(pages, response_key) @staticmethod - def _get_from_all_pages(pages: [], key:str): + def _get_from_all_pages(pages: [], key: str): resources = [] for page in pages: resources.extend(page[key]) @@ -22,5 +23,6 @@ def _get_from_all_pages(pages: [], key:str): return resources @staticmethod - def get_client(service: str, region: str): - return AWSFacadeUtils._clients.setdefault((service, region), boto3.client(service, region_name=region)) + async def get_client(service: str, region: str): + client = await run_concurrently(lambda: boto3.client(service, region_name=region)) + return AWSFacadeUtils._clients.setdefault((service, region), client) From 90ee63eb1139025a5770e1a22ec081e5524603a3 Mon Sep 17 00:00:00 2001 From: misg Date: Tue, 5 Mar 2019 16:12:32 -0500 Subject: [PATCH 252/667] Update EC2 service to use the new async AWS facade. --- ScoutSuite/providers/aws/resources/ec2/ami.py | 3 +-- ScoutSuite/providers/aws/resources/ec2/instances.py | 11 +++++------ .../providers/aws/resources/ec2/networkinterfaces.py | 9 ++++----- .../providers/aws/resources/ec2/securitygroups.py | 5 +---- ScoutSuite/providers/aws/resources/ec2/snapshots.py | 6 +----- ScoutSuite/providers/aws/resources/ec2/volumes.py | 7 +++---- ScoutSuite/providers/aws/resources/ec2/vpcs.py | 3 +-- 7 files changed, 16 insertions(+), 28 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/ec2/ami.py b/ScoutSuite/providers/aws/resources/ec2/ami.py index 7ba4c7f6a..f5312b8ca 100644 --- a/ScoutSuite/providers/aws/resources/ec2/ami.py +++ b/ScoutSuite/providers/aws/resources/ec2/ami.py @@ -1,10 +1,9 @@ from ScoutSuite.providers.aws.resources.resources import AWSResources -from ScoutSuite.providers.aws.facade.facade import AWSFacade class AmazonMachineImages(AWSResources): async def fetch_all(self, **kwargs): - raw_images = self.facade.ec2.get_images(self.scope['region'], self.scope['owner_id']) + raw_images = await self.facade.ec2.get_images(self.scope['region'], self.scope['owner_id']) for raw_image in raw_images: name, resource = self._parse_image(raw_image) self[name] = resource diff --git a/ScoutSuite/providers/aws/resources/ec2/instances.py b/ScoutSuite/providers/aws/resources/ec2/instances.py index c5b9bf709..f173cd63f 100644 --- a/ScoutSuite/providers/aws/resources/ec2/instances.py +++ b/ScoutSuite/providers/aws/resources/ec2/instances.py @@ -1,23 +1,22 @@ from ScoutSuite.providers.aws.resources.resources import AWSResources -from ScoutSuite.providers.aws.facade.facade import AWSFacade from ScoutSuite.providers.aws.aws import get_name -from ScoutSuite.providers.aws.utils import ec2_classic, get_keys +from ScoutSuite.providers.aws.utils import get_keys class EC2Instances(AWSResources): async def fetch_all(self, **kwargs): - raw_instances = self.facade.ec2.get_instances(self.scope['region'], self.scope['vpc']) + raw_instances = await self.facade.ec2.get_instances(self.scope['region'], self.scope['vpc']) for raw_instance in raw_instances: - name, resource = self._parse_instance(raw_instance) + name, resource = await self._parse_instance(raw_instance) self[name] = resource - def _parse_instance(self, raw_instance): + async def _parse_instance(self, raw_instance): instance = {} id = raw_instance['InstanceId'] instance['id'] = id instance['reservation_id'] = raw_instance['ReservationId'] instance['monitoring_enabled'] = raw_instance['Monitoring']['State'] == 'enabled' - instance['user_data'] = self.facade.ec2.get_instance_user_data(self.scope['region'], id) + instance['user_data'] = await self.facade.ec2.get_instance_user_data(self.scope['region'], id) get_name(raw_instance, instance, 'InstanceId') get_keys(raw_instance, instance, ['KeyName', 'LaunchTime', 'InstanceType', 'State', 'IamInstanceProfile', 'SubnetId']) diff --git a/ScoutSuite/providers/aws/resources/ec2/networkinterfaces.py b/ScoutSuite/providers/aws/resources/ec2/networkinterfaces.py index 63408ddab..ce5bc3a03 100644 --- a/ScoutSuite/providers/aws/resources/ec2/networkinterfaces.py +++ b/ScoutSuite/providers/aws/resources/ec2/networkinterfaces.py @@ -1,14 +1,13 @@ from ScoutSuite.providers.aws.resources.resources import AWSResources -from ScoutSuite.providers.aws.facade.facade import AWSFacade class NetworkInterfaces(AWSResources): async def fetch_all(self, **kwargs): - raw_security_groups = self.facade.ec2.get_network_interfaces(self.scope['region'], self.scope['vpc']) + raw_security_groups = await self.facade.ec2.get_network_interfaces(self.scope['region'], self.scope['vpc']) for raw_security_groups in raw_security_groups: name, resource = self._parse_network_interface(raw_security_groups) self[name] = resource - def _parse_network_interface(self, raw_network_interace): - raw_network_interace['name'] = raw_network_interace['NetworkInterfaceId'] - return raw_network_interace['NetworkInterfaceId'], raw_network_interace + def _parse_network_interface(self, raw_network_interface): + raw_network_interface['name'] = raw_network_interface['NetworkInterfaceId'] + return raw_network_interface['NetworkInterfaceId'], raw_network_interface diff --git a/ScoutSuite/providers/aws/resources/ec2/securitygroups.py b/ScoutSuite/providers/aws/resources/ec2/securitygroups.py index c59b87129..d5009f407 100644 --- a/ScoutSuite/providers/aws/resources/ec2/securitygroups.py +++ b/ScoutSuite/providers/aws/resources/ec2/securitygroups.py @@ -1,7 +1,4 @@ from ScoutSuite.providers.aws.resources.resources import AWSResources -from ScoutSuite.providers.aws.facade.facade import AWSFacade -from ScoutSuite.providers.aws.aws import get_name -from ScoutSuite.providers.aws.utils import ec2_classic, get_keys from ScoutSuite.utils import manage_dictionary from ScoutSuite.core.fs import load_data @@ -10,7 +7,7 @@ class SecurityGroups(AWSResources): icmp_message_types_dict = load_data('icmp_message_types.json', 'icmp_message_types') async def fetch_all(self, **kwargs): - raw_security_groups = self.facade.ec2.get_security_groups(self.scope['region'], self.scope['vpc']) + raw_security_groups = await self.facade.ec2.get_security_groups(self.scope['region'], self.scope['vpc']) for raw_security_groups in raw_security_groups: name, resource = self._parse_security_group(raw_security_groups) self[name] = resource diff --git a/ScoutSuite/providers/aws/resources/ec2/snapshots.py b/ScoutSuite/providers/aws/resources/ec2/snapshots.py index 1606bb8b9..9c36ca5ce 100644 --- a/ScoutSuite/providers/aws/resources/ec2/snapshots.py +++ b/ScoutSuite/providers/aws/resources/ec2/snapshots.py @@ -1,18 +1,14 @@ from ScoutSuite.providers.aws.resources.resources import AWSResources -from ScoutSuite.providers.aws.facade.facade import AWSFacade from ScoutSuite.providers.aws.aws import get_name class Snapshots(AWSResources): async def fetch_all(self, **kwargs): - raw_snapshots = self.facade.ec2.get_snapshots(self.scope['region'], self.scope['owner_id']) + raw_snapshots = await self.facade.ec2.get_snapshots(self.scope['region'], self.scope['owner_id']) for raw_snapshot in raw_snapshots: name, resource = self._parse_snapshot(raw_snapshot) self[name] = resource - async def get_resources_from_api(self): - return self.facade.ec2.get_snapshots(self.scope['region'], self.scope['owner_id']) - def _parse_snapshot(self, raw_snapshot): raw_snapshot['id'] = raw_snapshot.pop('SnapshotId') raw_snapshot['name'] = get_name(raw_snapshot, raw_snapshot, 'id') diff --git a/ScoutSuite/providers/aws/resources/ec2/volumes.py b/ScoutSuite/providers/aws/resources/ec2/volumes.py index 16970fcf5..c36367b8a 100644 --- a/ScoutSuite/providers/aws/resources/ec2/volumes.py +++ b/ScoutSuite/providers/aws/resources/ec2/volumes.py @@ -1,16 +1,15 @@ from ScoutSuite.providers.aws.resources.resources import AWSResources -from ScoutSuite.providers.aws.facade.facade import AWSFacade from ScoutSuite.providers.aws.aws import get_name class Volumes(AWSResources): async def fetch_all(self, **kwargs): - raw_volumes = self.facade.ec2.get_volumes(self.scope['region']) + raw_volumes = await self.facade.ec2.get_volumes(self.scope['region']) for raw_volume in raw_volumes: - name, resource = self._parse_volumes(raw_volume) + name, resource = self._parse_volume(raw_volume) self[name] = resource - def _parse_volumes(self, raw_volume): + def _parse_volume(self, raw_volume): raw_volume['id'] = raw_volume.pop('VolumeId') raw_volume['name'] = get_name(raw_volume, raw_volume, 'id') return raw_volume['id'], raw_volume diff --git a/ScoutSuite/providers/aws/resources/ec2/vpcs.py b/ScoutSuite/providers/aws/resources/ec2/vpcs.py index 7b5c6ceeb..fab0e083b 100644 --- a/ScoutSuite/providers/aws/resources/ec2/vpcs.py +++ b/ScoutSuite/providers/aws/resources/ec2/vpcs.py @@ -1,5 +1,4 @@ from ScoutSuite.providers.aws.resources.resources import AWSCompositeResources -from ScoutSuite.providers.aws.facade.facade import AWSFacade from ScoutSuite.providers.aws.resources.ec2.instances import EC2Instances from ScoutSuite.providers.aws.resources.ec2.securitygroups import SecurityGroups from ScoutSuite.providers.aws.resources.ec2.networkinterfaces import NetworkInterfaces @@ -13,7 +12,7 @@ class Vpcs(AWSCompositeResources): ] async def fetch_all(self, **kwargs): - vpcs = self.facade.ec2.get_vpcs(self.scope['region']) + vpcs = await self.facade.ec2.get_vpcs(self.scope['region']) for vpc in vpcs: name, resource = self._parse_vpc(vpc) self[name] = resource From 3b42f46a16cdc65958dd67db9d999c9c50ef9f7e Mon Sep 17 00:00:00 2001 From: misg Date: Tue, 5 Mar 2019 16:13:05 -0500 Subject: [PATCH 253/667] Update AWSLambda service to use the new async AWS facade. --- ScoutSuite/providers/aws/resources/awslambda/service.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/awslambda/service.py b/ScoutSuite/providers/aws/resources/awslambda/service.py index 7ec5fbb35..f50c4cbc1 100644 --- a/ScoutSuite/providers/aws/resources/awslambda/service.py +++ b/ScoutSuite/providers/aws/resources/awslambda/service.py @@ -1,18 +1,17 @@ from ScoutSuite.providers.aws.resources.regions import Regions from ScoutSuite.providers.aws.resources.resources import AWSResources -from ScoutSuite.providers.aws.facade.facade import AWSFacade class RegionalLambdas(AWSResources): async def fetch_all(self, **kwargs): - raw_functions = self.facade.awslambda.get_functions(self.scope['region']) + raw_functions = await self.facade.awslambda.get_functions(self.scope['region']) for raw_function in raw_functions: name, resource = self._parse_function(raw_function) self[name] = resource def _parse_function(self, raw_function): raw_function['name'] = raw_function.pop('FunctionName') - return (raw_function['name'], raw_function) + return raw_function['name'], raw_function class Lambdas(Regions): From dc3630f333172c93999dd9a7d40864235c8af6db Mon Sep 17 00:00:00 2001 From: misg Date: Tue, 5 Mar 2019 16:13:49 -0500 Subject: [PATCH 254/667] Minor changes. --- ScoutSuite/providers/aws/resources/regions.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/regions.py b/ScoutSuite/providers/aws/resources/regions.py index 1f0b3a7c0..b3e9ea274 100644 --- a/ScoutSuite/providers/aws/resources/regions.py +++ b/ScoutSuite/providers/aws/resources/regions.py @@ -1,7 +1,9 @@ +import abc + from ScoutSuite.providers.aws.aws import get_aws_account_id from ScoutSuite.providers.aws.resources.resources import AWSCompositeResources from ScoutSuite.providers.aws.facade.facade import AWSFacade -import abc + class Regions(AWSCompositeResources, metaclass=abc.ABCMeta): def __init__(self, service): @@ -10,7 +12,6 @@ def __init__(self, service): self.facade = AWSFacade() async def fetch_all(self, credentials, regions=None, partition_name='aws'): - self['regions'] = {} for region in await self.facade.build_region_list(self.service, regions, partition_name): self['regions'][region] = { From d2da4cdc09bb800d9208a07a1e09c0a961f37108 Mon Sep 17 00:00:00 2001 From: misg Date: Tue, 5 Mar 2019 16:14:52 -0500 Subject: [PATCH 255/667] Add an async facade for Azure SQL database service. --- .../providers/azure/facade/sqldatabase.py | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 ScoutSuite/providers/azure/facade/sqldatabase.py diff --git a/ScoutSuite/providers/azure/facade/sqldatabase.py b/ScoutSuite/providers/azure/facade/sqldatabase.py new file mode 100644 index 000000000..2269d5558 --- /dev/null +++ b/ScoutSuite/providers/azure/facade/sqldatabase.py @@ -0,0 +1,44 @@ +from azure.mgmt.sql import SqlManagementClient +from ScoutSuite.providers.utils import run_concurrently + + +class SQLDatabaseFacade: + def __init__(self, credentials, subscription_id): + self._client = SqlManagementClient(credentials, subscription_id) + + async def get_database_blob_auditing_policies(self, resource_group_name, server_name, database_name): + return await run_concurrently( + lambda: self._client.database_blob_auditing_policies.get( + resource_group_name, server_name, database_name) + ) + + async def get_database_threat_detection_policies(self, resource_group_name, server_name, database_name): + return await run_concurrently( + lambda: self._client.database_threat_detection_policies.get( + resource_group_name, server_name, database_name) + ) + + async def get_databases(self, resource_group_name, server_name): + return await run_concurrently( + lambda: self._client.databases.list_by_server(resource_group_name, server_name) + ) + + async def get_database_replication_links(self, resource_group_name, server_name, database_name): + return await run_concurrently( + lambda: list(self._client.replication_links.list_by_database( + resource_group_name, server_name, database_name)) + ) + + async def get_server_azure_ad_administrators(self, resource_group_name, server_name): + return await run_concurrently( + lambda: self._client.server_azure_ad_administrators.get(resource_group_name, server_name) + ) + + async def get_servers(self): + return await run_concurrently(self._client.servers.list) + + async def get_database_transparent_data_encryptions(self, resource_group_name, server_name, database_name): + return await run_concurrently( + lambda: self._client.transparent_data_encryptions.get( + resource_group_name, server_name, database_name) + ) From fc08156299d973f5492dc0542f53c51f5647c773 Mon Sep 17 00:00:00 2001 From: misg Date: Tue, 5 Mar 2019 16:15:33 -0500 Subject: [PATCH 256/667] Improve async support. --- ScoutSuite/providers/aws/resources/resources.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/resources.py b/ScoutSuite/providers/aws/resources/resources.py index 1741885ff..0e47025e0 100644 --- a/ScoutSuite/providers/aws/resources/resources.py +++ b/ScoutSuite/providers/aws/resources/resources.py @@ -1,9 +1,10 @@ - """This module provides implementations for Resources and CompositeResources for AWS.""" +import abc +import asyncio + from ScoutSuite.providers.base.configs.resources import Resources, CompositeResources from ScoutSuite.providers.aws.facade.facade import AWSFacade -import abc class AWSResources(Resources, metaclass=abc.ABCMeta): @@ -38,12 +39,14 @@ async def _fetch_children(self, parent: object, scope: dict): :param scope: The scope passed to the children constructors """ - for child_class, child_name in self._children: - child = child_class(scope) - await child.fetch_all() - + children = [(child_class(scope), child_name) for (child_class, child_name) in self._children] + # fetch all children concurrently: + await asyncio.wait({asyncio.create_task(child.fetch_all()) for (child, _) in children}) + # update parent content: + for child, child_name in children: if parent.get(child_name) is None: parent[child_name] = {} parent[child_name].update(child) parent[child_name + '_count'] = len(child) + From f88807587168d7ac187870c98d63f84d6c083661 Mon Sep 17 00:00:00 2001 From: misg Date: Tue, 5 Mar 2019 16:16:10 -0500 Subject: [PATCH 257/667] Update sqldatabase resources to use the async facade. --- .../sqldatabase/database_blob_auditing_policies.py | 3 +-- .../sqldatabase/database_threat_detection_policies.py | 3 +-- .../providers/azure/resources/sqldatabase/databases.py | 2 +- .../azure/resources/sqldatabase/replication_links.py | 5 ++--- .../sqldatabase/server_azure_ad_administrators.py | 3 +-- .../providers/azure/resources/sqldatabase/servers.py | 7 +++---- .../resources/sqldatabase/transparent_data_encryptions.py | 3 +-- 7 files changed, 10 insertions(+), 16 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sqldatabase/database_blob_auditing_policies.py b/ScoutSuite/providers/azure/resources/sqldatabase/database_blob_auditing_policies.py index cf9d9dda5..7661f4c35 100644 --- a/ScoutSuite/providers/azure/resources/sqldatabase/database_blob_auditing_policies.py +++ b/ScoutSuite/providers/azure/resources/sqldatabase/database_blob_auditing_policies.py @@ -9,9 +9,8 @@ def __init__(self, resource_group_name, server_name, database_name, facade): self.database_name = database_name self.facade = facade - # TODO: make it really async. async def fetch_all(self): - policies = self.facade.database_blob_auditing_policies.get( + policies = await self.facade.get_database_blob_auditing_policies( self.resource_group_name, self.server_name, self.database_name) self._parse_policies(policies) diff --git a/ScoutSuite/providers/azure/resources/sqldatabase/database_threat_detection_policies.py b/ScoutSuite/providers/azure/resources/sqldatabase/database_threat_detection_policies.py index a758d4496..c2865a5a5 100644 --- a/ScoutSuite/providers/azure/resources/sqldatabase/database_threat_detection_policies.py +++ b/ScoutSuite/providers/azure/resources/sqldatabase/database_threat_detection_policies.py @@ -9,9 +9,8 @@ def __init__(self, resource_group_name, server_name, database_name, facade): self.database_name = database_name self.facade = facade - # TODO: make it really async. async def fetch_all(self): - policies = self.facade.database_threat_detection_policies.get( + policies = await self.facade.get_database_threat_detection_policies( self.resource_group_name, self.server_name, self.database_name) self._parse_policies(policies) diff --git a/ScoutSuite/providers/azure/resources/sqldatabase/databases.py b/ScoutSuite/providers/azure/resources/sqldatabase/databases.py index f2a28dae0..c474a6e72 100644 --- a/ScoutSuite/providers/azure/resources/sqldatabase/databases.py +++ b/ScoutSuite/providers/azure/resources/sqldatabase/databases.py @@ -23,7 +23,7 @@ def __init__(self, resource_group_name, server_name, facade): async def fetch_all(self): self['databases'] = {} - for db in self.facade.databases.list_by_server(self.resource_group_name, self.server_name): + for db in await self.facade.get_databases(self.resource_group_name, self.server_name): # We do not want to scan 'master' database which is auto-generated by Azure and read-only: if db.name == 'master': continue diff --git a/ScoutSuite/providers/azure/resources/sqldatabase/replication_links.py b/ScoutSuite/providers/azure/resources/sqldatabase/replication_links.py index 57f5bab72..bffe27ca8 100644 --- a/ScoutSuite/providers/azure/resources/sqldatabase/replication_links.py +++ b/ScoutSuite/providers/azure/resources/sqldatabase/replication_links.py @@ -9,10 +9,9 @@ def __init__(self, resource_group_name, server_name, database_name, facade): self.database_name = database_name self.facade = facade - # TODO: make it really async. async def fetch_all(self): - links = list(self.facade.replication_links.list_by_database( - self.resource_group_name, self.server_name, self.database_name)) + links = await self.facade.get_database_replication_links( + self.resource_group_name, self.server_name, self.database_name) self._parse_links(links) def _parse_links(self, links): diff --git a/ScoutSuite/providers/azure/resources/sqldatabase/server_azure_ad_administrators.py b/ScoutSuite/providers/azure/resources/sqldatabase/server_azure_ad_administrators.py index fc67436a9..1552fb98d 100644 --- a/ScoutSuite/providers/azure/resources/sqldatabase/server_azure_ad_administrators.py +++ b/ScoutSuite/providers/azure/resources/sqldatabase/server_azure_ad_administrators.py @@ -10,10 +10,9 @@ def __init__(self, resource_group_name, server_name, facade): self.server_name = server_name self.facade = facade - # TODO: make it really async. async def fetch_all(self): try: - self.facade.server_azure_ad_administrators.get(self.resource_group_name, self.server_name) + await self.facade.get_server_azure_ad_administrators(self.resource_group_name, self.server_name) self['ad_admin_configured'] = True except CloudError: # no ad admin configured returns a 404 error self['ad_admin_configured'] = False diff --git a/ScoutSuite/providers/azure/resources/sqldatabase/servers.py b/ScoutSuite/providers/azure/resources/sqldatabase/servers.py index 595298607..ff3361872 100644 --- a/ScoutSuite/providers/azure/resources/sqldatabase/servers.py +++ b/ScoutSuite/providers/azure/resources/sqldatabase/servers.py @@ -1,8 +1,7 @@ -from azure.mgmt.sql import SqlManagementClient - from ScoutSuite.providers.azure.resources.resources import AzureCompositeResources from ScoutSuite.providers.azure.utils import get_resource_group_name from ScoutSuite.providers.utils import get_non_provider_id +from ScoutSuite.providers.azure.facade.sqldatabase import SQLDatabaseFacade from .databases import Databases from .server_azure_ad_administrators import ServerAzureAdAdministrators @@ -17,10 +16,10 @@ class Servers(AzureCompositeResources): # TODO: make it really async. async def fetch_all(self, credentials, **kwargs): # TODO: build that facade somewhere else: - facade = SqlManagementClient(credentials.credentials, credentials.subscription_id) + facade = SQLDatabaseFacade(credentials.credentials, credentials.subscription_id) self['servers'] = {} - for server in facade.servers.list(): + for server in await facade.get_servers(): id = get_non_provider_id(server.id) resource_group_name = get_resource_group_name(server.id) diff --git a/ScoutSuite/providers/azure/resources/sqldatabase/transparent_data_encryptions.py b/ScoutSuite/providers/azure/resources/sqldatabase/transparent_data_encryptions.py index b4aa4c076..4fe946a4a 100644 --- a/ScoutSuite/providers/azure/resources/sqldatabase/transparent_data_encryptions.py +++ b/ScoutSuite/providers/azure/resources/sqldatabase/transparent_data_encryptions.py @@ -9,9 +9,8 @@ def __init__(self, resource_group_name, server_name, database_name, facade): self.database_name = database_name self.facade = facade - # TODO: make it really async. async def fetch_all(self): - encryptions = self.facade.transparent_data_encryptions.get( + encryptions = await self.facade.get_database_transparent_data_encryptions( self.resource_group_name, self.server_name, self.database_name) self._parse_encryptions(encryptions) From c362ff3508d0658768d6d00dcfefac5e3ce4e1c1 Mon Sep 17 00:00:00 2001 From: misg Date: Tue, 5 Mar 2019 16:16:27 -0500 Subject: [PATCH 258/667] Improve async support. --- ScoutSuite/providers/azure/resources/resources.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/resources.py b/ScoutSuite/providers/azure/resources/resources.py index 563c25ca7..b858f4dee 100644 --- a/ScoutSuite/providers/azure/resources/resources.py +++ b/ScoutSuite/providers/azure/resources/resources.py @@ -1,4 +1,5 @@ import abc +import asyncio from ScoutSuite.providers.base.configs.resources import CompositeResources @@ -6,7 +7,9 @@ class AzureCompositeResources(CompositeResources, metaclass=abc.ABCMeta): async def _fetch_children(self, parent, **kwargs): - for child_class in self._children: - child = child_class(**kwargs) - await child.fetch_all() + children = [child_class(**kwargs) for child_class in self._children] + # fetch all children concurrently: + await asyncio.wait({asyncio.create_task(child.fetch_all()) for child in children}) + # update parent content: + for child in children: parent.update(child) From c00aa87d8c3ddb562cb9d3488effaa35e0d31a24 Mon Sep 17 00:00:00 2001 From: misg Date: Tue, 5 Mar 2019 16:16:55 -0500 Subject: [PATCH 259/667] Add a thread pool executor. --- Scout.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Scout.py b/Scout.py index 83cb235dd..90f6963bc 100755 --- a/Scout.py +++ b/Scout.py @@ -4,10 +4,13 @@ import sys import asyncio +from concurrent.futures import ThreadPoolExecutor + from ScoutSuite.__main__ import main if __name__ == "__main__": loop = asyncio.get_event_loop() + loop.set_default_executor(ThreadPoolExecutor(max_workers=10)) loop.run_until_complete(main()) loop.close() sys.exit() From 6857f409a5b5b7b0e0f5af0880c1462e73660022 Mon Sep 17 00:00:00 2001 From: misg Date: Tue, 5 Mar 2019 16:31:36 -0500 Subject: [PATCH 260/667] Make cloudtrail facade async. --- ScoutSuite/providers/aws/facade/cloudtrail.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/cloudtrail.py b/ScoutSuite/providers/aws/facade/cloudtrail.py index 8836e7f63..45557dcd8 100644 --- a/ScoutSuite/providers/aws/facade/cloudtrail.py +++ b/ScoutSuite/providers/aws/facade/cloudtrail.py @@ -1,13 +1,20 @@ from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils +from ScoutSuite.providers.utils import run_concurrently class CloudTrailFacade: - def get_trails(self, region): - client = AWSFacadeUtils.get_client('cloudtrail', region) - trails = client.describe_trails()['trailList'] + async def get_trails(self, region): + client = await AWSFacadeUtils.get_client('cloudtrail', region) + trails = await run_concurrently( + lambda: client.describe_trails()['trailList'] + ) for trail in trails: - trail.update(client.get_trail_status(Name=trail['TrailARN'])) - trail['EventSelectors'] = client.get_event_selectors(TrailName=trail['TrailARN'])['EventSelectors'] + trail.update(await run_concurrently( + lambda: client.get_trail_status(Name=trail['TrailARN']) + )) + trail['EventSelectors'] = await run_concurrently( + lambda: client.get_event_selectors(TrailName=trail['TrailARN'])['EventSelectors'] + ) - return trails \ No newline at end of file + return trails From 0daa853c1df4807ee7513621064ed8e42679d45d Mon Sep 17 00:00:00 2001 From: misg Date: Tue, 5 Mar 2019 16:31:54 -0500 Subject: [PATCH 261/667] Update cloudtrail service to use its async facade. --- ScoutSuite/providers/aws/resources/cloudtrail/service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/cloudtrail/service.py b/ScoutSuite/providers/aws/resources/cloudtrail/service.py index 206ebd5cb..a2a962c12 100644 --- a/ScoutSuite/providers/aws/resources/cloudtrail/service.py +++ b/ScoutSuite/providers/aws/resources/cloudtrail/service.py @@ -1,13 +1,13 @@ from ScoutSuite.providers.aws.resources.regions import Regions from ScoutSuite.providers.aws.resources.resources import AWSResources -from ScoutSuite.providers.aws.facade.facade import AWSFacade from ScoutSuite.providers.utils import get_non_provider_id import time + class Trails(AWSResources): async def fetch_all(self, **kwargs): - raw_trails = self.facade.cloudtrail.get_trails(self.scope['region']) + raw_trails = await self.facade.cloudtrail.get_trails(self.scope['region']) for raw_trail in raw_trails: name, resource = self._parse_trail(raw_trail) self[name] = resource From c3def8230197fcae051eb05a127e254c689bc844 Mon Sep 17 00:00:00 2001 From: misg Date: Tue, 5 Mar 2019 16:51:10 -0500 Subject: [PATCH 262/667] Add TODO. --- Scout.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Scout.py b/Scout.py index 90f6963bc..1f1800adf 100755 --- a/Scout.py +++ b/Scout.py @@ -10,6 +10,7 @@ if __name__ == "__main__": loop = asyncio.get_event_loop() + # TODO: make max_workers parameterizable (through the thread_config cli argument) loop.set_default_executor(ThreadPoolExecutor(max_workers=10)) loop.run_until_complete(main()) loop.close() From a95e104aed475cb97062a155b75ded9081cb4149 Mon Sep 17 00:00:00 2001 From: misg Date: Tue, 5 Mar 2019 17:23:42 -0500 Subject: [PATCH 263/667] Use asyncio.ensure_future instead of asyncio.create_task for python 3.5 and 3.6 compatibility. --- ScoutSuite/providers/aws/resources/resources.py | 2 +- ScoutSuite/providers/azure/resources/resources.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/resources.py b/ScoutSuite/providers/aws/resources/resources.py index 0e47025e0..95361a738 100644 --- a/ScoutSuite/providers/aws/resources/resources.py +++ b/ScoutSuite/providers/aws/resources/resources.py @@ -41,7 +41,7 @@ async def _fetch_children(self, parent: object, scope: dict): children = [(child_class(scope), child_name) for (child_class, child_name) in self._children] # fetch all children concurrently: - await asyncio.wait({asyncio.create_task(child.fetch_all()) for (child, _) in children}) + await asyncio.wait({asyncio.ensure_future(child.fetch_all()) for (child, _) in children}) # update parent content: for child, child_name in children: if parent.get(child_name) is None: diff --git a/ScoutSuite/providers/azure/resources/resources.py b/ScoutSuite/providers/azure/resources/resources.py index 7d704559e..5523bd4e7 100644 --- a/ScoutSuite/providers/azure/resources/resources.py +++ b/ScoutSuite/providers/azure/resources/resources.py @@ -10,7 +10,7 @@ class AzureCompositeResources(CompositeResources, metaclass=abc.ABCMeta): async def _fetch_children(self, parent, **kwargs): children = [(child_class(**kwargs), child_name) for (child_class, child_name) in self._children] # fetch all children concurrently: - await asyncio.wait({asyncio.create_task(child.fetch_all()) for (child, _) in children}) + await asyncio.wait({asyncio.ensure_future(child.fetch_all()) for (child, _) in children}) # update parent content: for child, name in children: if name: From a889194ef0470b34a97cc0db8a595e8d6d32090c Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 6 Mar 2019 09:02:08 -0500 Subject: [PATCH 264/667] Implemented EFS facade --- ScoutSuite/providers/aws/facade/efs.py | 25 +++++++++++++++++++++++ ScoutSuite/providers/aws/facade/facade.py | 9 +++++--- 2 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 ScoutSuite/providers/aws/facade/efs.py diff --git a/ScoutSuite/providers/aws/facade/efs.py b/ScoutSuite/providers/aws/facade/efs.py new file mode 100644 index 000000000..7d7a042e9 --- /dev/null +++ b/ScoutSuite/providers/aws/facade/efs.py @@ -0,0 +1,25 @@ +from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils +from ScoutSuite.providers.aws.aws import handle_truncated_response + + +class EFSFacade: + def get_file_systems(self, region: str): + file_systems = AWSFacadeUtils.get_all_pages('efs', region, 'describe_file_systems', 'FileSystems') + + client = AWSFacadeUtils.get_client('efs', region) + for file_system in file_systems: + file_system_id = file_system['FileSystemId'] + file_system['Tags'] = client.describe_tags(FileSystemId=file_system_id)['Tags'] + + # Get mount targets + mount_targets = AWSFacadeUtils.get_all_pages('efs', region, 'describe_mount_targets', 'MountTargets', FileSystemId=file_system_id) + file_system['MountTargets'] = {} + for mt in mount_targets: + mount_target_id = mt['MountTargetId'] + file_system['MountTargets'][mount_target_id] = mt + + # Get security groups + security_groups = client.describe_mount_target_security_groups(MountTargetId=mount_target_id)['SecurityGroups'] + file_system['MountTargets'][mount_target_id]['SecurityGroups'] = security_groups + + return file_systems \ No newline at end of file diff --git a/ScoutSuite/providers/aws/facade/facade.py b/ScoutSuite/providers/aws/facade/facade.py index ae29dd9bc..b4404d29b 100644 --- a/ScoutSuite/providers/aws/facade/facade.py +++ b/ScoutSuite/providers/aws/facade/facade.py @@ -1,14 +1,17 @@ -from botocore.session import Session from collections import Counter -from ScoutSuite.providers.aws.facade.ec2 import EC2Facade +from botocore.session import Session from ScoutSuite.providers.aws.facade.awslambda import LambdaFacade from ScoutSuite.providers.aws.facade.cloudtrail import CloudTrailFacade +from ScoutSuite.providers.aws.facade.ec2 import EC2Facade +from ScoutSuite.providers.aws.facade.efs import EFSFacade + class AWSFacade(object): def __init__(self): self.ec2 = EC2Facade() self.awslambda = LambdaFacade() self.cloudtrail = CloudTrailFacade() + self.efs = EFSFacade() async def build_region_list(self, service: str, chosen_regions=None, partition_name='aws'): @@ -23,4 +26,4 @@ async def build_region_list(self, service: str, chosen_regions=None, partition_n if chosen_regions: return list((Counter(regions) & Counter(chosen_regions)).elements()) else: - return regions \ No newline at end of file + return regions From a295d00eb34a94cec1bf27ae9e5227e54e6a66fc Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 6 Mar 2019 09:02:17 -0500 Subject: [PATCH 265/667] Migrated EFS to the new architecture --- .../providers/aws/resources/efs/service.py | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 ScoutSuite/providers/aws/resources/efs/service.py diff --git a/ScoutSuite/providers/aws/resources/efs/service.py b/ScoutSuite/providers/aws/resources/efs/service.py new file mode 100644 index 000000000..3917a9ecd --- /dev/null +++ b/ScoutSuite/providers/aws/resources/efs/service.py @@ -0,0 +1,30 @@ +from ScoutSuite.providers.aws.resources.regions import Regions +from ScoutSuite.providers.aws.resources.resources import AWSResources +from ScoutSuite.providers.aws.facade.facade import AWSFacade + + +class FileSystems(AWSResources): + async def fetch_all(self, **kwargs): + raw_file_systems = self.facade.efs.get_file_systems(self.scope['region']) + for raw_file_system in raw_file_systems: + name, resource = self._parse(raw_file_system) + self[name] = resource + + def _parse(self, raw_file_system): + fs_id = raw_file_system.pop('FileSystemId') + raw_file_system['name'] = raw_file_system.pop('Name') if 'Name' in raw_file_system else None + + # Get tags + raw_file_system['tags'] = raw_file_system.pop('Tags') + + return fs_id, raw_file_system + + + +class EFS(Regions): + _children = [ + (FileSystems, 'filesystems') + ] + + def __init__(self): + super(EFS, self).__init__('efs') From 107c21dcef9097e35ec121cd85372af047186809 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 6 Mar 2019 09:02:26 -0500 Subject: [PATCH 266/667] Integrated new architecture --- ScoutSuite/providers/aws/configs/services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index 19ea0954b..5000a6944 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -6,7 +6,7 @@ from ScoutSuite.providers.aws.services.cloudwatch import CloudWatchConfig from ScoutSuite.providers.aws.services.directconnect import DirectConnectConfig from ScoutSuite.providers.aws.resources.ec2.service import EC2 -from ScoutSuite.providers.aws.services.efs import EFSConfig +from ScoutSuite.providers.aws.resources.efs.service import EFS from ScoutSuite.providers.aws.services.elasticache import ElastiCacheConfig from ScoutSuite.providers.aws.services.elb import ELBConfig from ScoutSuite.providers.aws.services.elbv2 import ELBv2Config @@ -59,7 +59,7 @@ def __init__(self, metadata=None, thread_config=4, **kwargs): self.cloudwatch = CloudWatchConfig(metadata['management']['cloudwatch'], thread_config) self.directconnect = DirectConnectConfig(metadata['network']['directconnect'], thread_config) self.ec2 = EC2() - self.efs = EFSConfig(metadata['storage']['efs'], thread_config) + self.efs = EFS() self.elasticache = ElastiCacheConfig(metadata['database']['elasticache'], thread_config) self.elb = ELBConfig(metadata['compute']['elb'], thread_config) self.elbv2 = ELBv2Config(metadata['compute']['elbv2'], thread_config) From 775822401683e256d29f34ea38b85b47de207de7 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 6 Mar 2019 12:31:07 -0500 Subject: [PATCH 267/667] Made get_client thread-safe-r --- ScoutSuite/providers/aws/facade/utils.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/utils.py b/ScoutSuite/providers/aws/facade/utils.py index da273f72e..4c94172d2 100644 --- a/ScoutSuite/providers/aws/facade/utils.py +++ b/ScoutSuite/providers/aws/facade/utils.py @@ -1,10 +1,11 @@ import boto3 from ScoutSuite.providers.utils import run_concurrently - +from threading import Lock # TODO: Add docstrings class AWSFacadeUtils: + _get_client_lock = Lock() _clients = {} @staticmethod @@ -24,5 +25,9 @@ def _get_from_all_pages(pages: [], key: str): @staticmethod async def get_client(service: str, region: str): - client = await run_concurrently(lambda: boto3.client(service, region_name=region)) - return AWSFacadeUtils._clients.setdefault((service, region), client) + client_key = (service, region) + + if client_key not in AWSFacadeUtils._clients: + AWSFacadeUtils._clients[client_key] = boto3.client(service, region_name=region) + + return AWSFacadeUtils._clients[client_key] From 87e842f0816c4215cfdc9f83489f277e0820c84b Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 6 Mar 2019 14:21:10 -0500 Subject: [PATCH 268/667] Reduced number of api calls --- ScoutSuite/providers/aws/resources/regions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/resources/regions.py b/ScoutSuite/providers/aws/resources/regions.py index b3e9ea274..89d6a1add 100644 --- a/ScoutSuite/providers/aws/resources/regions.py +++ b/ScoutSuite/providers/aws/resources/regions.py @@ -13,6 +13,7 @@ def __init__(self, service): async def fetch_all(self, credentials, regions=None, partition_name='aws'): self['regions'] = {} + account_id = get_aws_account_id(credentials) for region in await self.facade.build_region_list(self.service, regions, partition_name): self['regions'][region] = { 'id': region, @@ -20,7 +21,7 @@ async def fetch_all(self, credentials, regions=None, partition_name='aws'): 'name': region } - await self._fetch_children(self['regions'][region], {'region': region, 'owner_id': get_aws_account_id(credentials)}) + await self._fetch_children(self['regions'][region], {'region': region, 'owner_id': account_id}) self._set_counts() From ea8da4909437549a77525558e4556cadf1fcae7a Mon Sep 17 00:00:00 2001 From: misg Date: Wed, 6 Mar 2019 16:09:26 -0500 Subject: [PATCH 269/667] Make fetching vpcs concurrent. --- ScoutSuite/providers/aws/resources/ec2/vpcs.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/ec2/vpcs.py b/ScoutSuite/providers/aws/resources/ec2/vpcs.py index fab0e083b..cbc770392 100644 --- a/ScoutSuite/providers/aws/resources/ec2/vpcs.py +++ b/ScoutSuite/providers/aws/resources/ec2/vpcs.py @@ -1,3 +1,5 @@ +import asyncio + from ScoutSuite.providers.aws.resources.resources import AWSCompositeResources from ScoutSuite.providers.aws.resources.ec2.instances import EC2Instances from ScoutSuite.providers.aws.resources.ec2.securitygroups import SecurityGroups @@ -17,9 +19,15 @@ async def fetch_all(self, **kwargs): name, resource = self._parse_vpc(vpc) self[name] = resource - for vpc in self: - scope = {'region': self.scope['region'], 'vpc': vpc} - await self._fetch_children(self[vpc], scope=scope) + tasks = { + asyncio.ensure_future( + self._fetch_children( + self[vpc], + {'region': self.scope['region'], 'vpc': vpc} + ) + ) for vpc in self + } + await asyncio.wait(tasks) def _parse_vpc(self, vpc): return vpc['VpcId'], {} From fe76ba504626dd7341eb062234f0831a1d878386 Mon Sep 17 00:00:00 2001 From: misg Date: Wed, 6 Mar 2019 16:10:00 -0500 Subject: [PATCH 270/667] Make fetching regions concurrent. --- ScoutSuite/providers/aws/resources/regions.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/resources/regions.py b/ScoutSuite/providers/aws/resources/regions.py index b3e9ea274..e445978d8 100644 --- a/ScoutSuite/providers/aws/resources/regions.py +++ b/ScoutSuite/providers/aws/resources/regions.py @@ -1,4 +1,5 @@ import abc +import asyncio from ScoutSuite.providers.aws.aws import get_aws_account_id from ScoutSuite.providers.aws.resources.resources import AWSCompositeResources @@ -20,7 +21,15 @@ async def fetch_all(self, credentials, regions=None, partition_name='aws'): 'name': region } - await self._fetch_children(self['regions'][region], {'region': region, 'owner_id': get_aws_account_id(credentials)}) + tasks = { + asyncio.ensure_future( + self._fetch_children( + self['regions'][region], + {'region': region, 'owner_id': get_aws_account_id(credentials)} + ) + ) for region in self['regions'] + } + await asyncio.wait(tasks) self._set_counts() From 446171c0c8dc36003cf2f61d6b59acd6005b1a70 Mon Sep 17 00:00:00 2001 From: misg Date: Wed, 6 Mar 2019 16:11:16 -0500 Subject: [PATCH 271/667] Make get_client non concurrent and make get_all_pages really concurrent. --- ScoutSuite/providers/aws/facade/utils.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/utils.py b/ScoutSuite/providers/aws/facade/utils.py index da273f72e..7550428e2 100644 --- a/ScoutSuite/providers/aws/facade/utils.py +++ b/ScoutSuite/providers/aws/facade/utils.py @@ -9,20 +9,23 @@ class AWSFacadeUtils: @staticmethod async def get_all_pages(service: str, region: str, paginator_name: str, response_key: str, **paginator_args): - client = await AWSFacadeUtils.get_client(service, region) - pages = await run_concurrently(lambda: client.get_paginator(paginator_name).paginate(**paginator_args)) + client = AWSFacadeUtils.get_client(service, region) + # Building a paginator doesn't require any API call so no need to do it concurrently: + paginator = client.get_paginator(paginator_name).paginate(**paginator_args) - return AWSFacadeUtils._get_from_all_pages(pages, response_key) + # Getting all pages from a paginator requires API calls so we need to do it concurrently: + return await run_concurrently(lambda: AWSFacadeUtils._get_all_pages_from_paginator(paginator, response_key)) @staticmethod - def _get_from_all_pages(pages: [], key: str): + def _get_all_pages_from_paginator(paginator, key): resources = [] - for page in pages: + # There's an API call hidden behind each iteration: + for page in paginator: resources.extend(page[key]) return resources @staticmethod - async def get_client(service: str, region: str): - client = await run_concurrently(lambda: boto3.client(service, region_name=region)) + def get_client(service: str, region: str): + client = boto3.client(service, region_name=region) return AWSFacadeUtils._clients.setdefault((service, region), client) From 550875841d53b6cc362076ca1f15efc85d0b28b8 Mon Sep 17 00:00:00 2001 From: misg Date: Wed, 6 Mar 2019 16:12:01 -0500 Subject: [PATCH 272/667] Update facades accordingly to the previous changes. --- ScoutSuite/providers/aws/facade/cloudtrail.py | 2 +- ScoutSuite/providers/aws/facade/ec2.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/cloudtrail.py b/ScoutSuite/providers/aws/facade/cloudtrail.py index 45557dcd8..58bafc2c3 100644 --- a/ScoutSuite/providers/aws/facade/cloudtrail.py +++ b/ScoutSuite/providers/aws/facade/cloudtrail.py @@ -4,7 +4,7 @@ class CloudTrailFacade: async def get_trails(self, region): - client = await AWSFacadeUtils.get_client('cloudtrail', region) + client = AWSFacadeUtils.get_client('cloudtrail', region) trails = await run_concurrently( lambda: client.describe_trails()['trailList'] ) diff --git a/ScoutSuite/providers/aws/facade/ec2.py b/ScoutSuite/providers/aws/facade/ec2.py index febb1da69..e9939f601 100644 --- a/ScoutSuite/providers/aws/facade/ec2.py +++ b/ScoutSuite/providers/aws/facade/ec2.py @@ -7,7 +7,7 @@ class EC2Facade: async def get_instance_user_data(self, region: str, instance_id: str): - ec2_client = await AWSFacadeUtils.get_client('ec2', region) + ec2_client = AWSFacadeUtils.get_client('ec2', region) user_data_response = await run_concurrently( lambda: ec2_client.describe_instance_attribute(Attribute='userData', InstanceId=instance_id)) @@ -40,7 +40,7 @@ async def get_vpcs(self, region): async def get_images(self, region, owner_id): filters = [{'Name': 'owner-id', 'Values': [owner_id]}] - client = await AWSFacadeUtils.get_client('ec2', region) + client = AWSFacadeUtils.get_client('ec2', region) response = await run_concurrently(lambda: client.describe_images(Filters=filters)) return response['Images'] @@ -58,7 +58,7 @@ async def get_snapshots(self, region, owner_id): snapshots = await AWSFacadeUtils.get_all_pages( 'ec2', region, 'describe_snapshots', 'Snapshots', Filters=filters) - ec2_client = await AWSFacadeUtils.get_client('ec2', region) + ec2_client = AWSFacadeUtils.get_client('ec2', region) for snapshot in snapshots: snapshot['CreateVolumePermissions'] = await run_concurrently(lambda: ec2_client.describe_snapshot_attribute( Attribute='createVolumePermission', From 36a70746b2f5fd0d1d9b87f152714346d262395c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sgha=C3=AFer?= Date: Wed, 6 Mar 2019 16:33:12 -0500 Subject: [PATCH 273/667] Made a bunch of stuff async Co-Authored-By: Aboisier --- .../providers/aws/facade/cloudformation.py | 19 +++++++++++++------ .../aws/resources/cloudformation/service.py | 2 +- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/cloudformation.py b/ScoutSuite/providers/aws/facade/cloudformation.py index 655433b90..7dc4b2981 100644 --- a/ScoutSuite/providers/aws/facade/cloudformation.py +++ b/ScoutSuite/providers/aws/facade/cloudformation.py @@ -1,19 +1,26 @@ import json from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils +from ScoutSuite.providers.utils import run_concurrently class CloudFormation: - def get_stacks(self, region: str): - stacks = AWSFacadeUtils.get_all_pages('cloudformation', region, 'list_stacks', 'StackSummaries') + async def get_stacks(self, region: str): + stacks = await AWSFacadeUtils.get_all_pages('cloudformation', region, 'list_stacks', 'StackSummaries') - client = AWSFacadeUtils.get_client('cloudformation', region) + client = await AWSFacadeUtils.get_client('cloudformation', region) for stack in stacks: stack_name = stack['StackName'] - stack_description = client.describe_stacks(StackName=stack_name)['Stacks'][0] + stack_description = await run_concurrently( + lambda: client.describe_stacks(StackName=stack_name)['Stacks'][0] + ) stack.update(stack_description) - stack['template'] = client.get_template(StackName=stack_name)['TemplateBody'] - stack_policy = client.get_stack_policy(StackName=stack_name) + stack['template'] = await run_concurrently( + lambda: client.get_template(StackName=stack_name)['TemplateBody'] + ) + stack_policy = await run_concurrently( + lambda: client.get_stack_policy(StackName=stack_name) + ) if 'StackPolicyBody' in stack_policy: stack['policy'] = json.loads(stack_policy['StackPolicyBody']) diff --git a/ScoutSuite/providers/aws/resources/cloudformation/service.py b/ScoutSuite/providers/aws/resources/cloudformation/service.py index a114ae426..0479ef499 100644 --- a/ScoutSuite/providers/aws/resources/cloudformation/service.py +++ b/ScoutSuite/providers/aws/resources/cloudformation/service.py @@ -5,7 +5,7 @@ class Stacks(AWSResources): async def fetch_all(self, **kwargs): - raw_stacks = self.facade.cloudformation.get_stacks(self.scope['region']) + raw_stacks = await self.facade.cloudformation.get_stacks(self.scope['region']) for raw_stack in raw_stacks: name, stack = self._parse_stack(raw_stack) self[name] = stack From 2f16b81341854acf502f0a6010a947a1748aafd3 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 6 Mar 2019 16:43:33 -0500 Subject: [PATCH 274/667] Removed old implementation --- ScoutSuite/providers/aws/services/efs.py | 61 ------------------------ 1 file changed, 61 deletions(-) delete mode 100644 ScoutSuite/providers/aws/services/efs.py diff --git a/ScoutSuite/providers/aws/services/efs.py b/ScoutSuite/providers/aws/services/efs.py deleted file mode 100644 index ab4e6186a..000000000 --- a/ScoutSuite/providers/aws/services/efs.py +++ /dev/null @@ -1,61 +0,0 @@ -# -*- coding: utf-8 -*- -""" -EFS-related classes and functions -""" -from ScoutSuite.providers.aws.aws import handle_truncated_response - -from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients - - -######################################## -# EFSRegionConfig -######################################## - -class EFSRegionConfig(RegionConfig): - """ - EFS configuration for a single AWS region - """ - file_systems = {} - - def parse_file_system(self, global_params, region, file_system): - """ - - :param global_params: - :param region: Name of the AWS region - :param file_system: - :return: - """ - fs_id = file_system.pop('FileSystemId') - if 'Name' in file_system: - file_system['name'] = file_system.pop('Name') - else: - file_system['name'] = None - # Get tags - file_system['tags'] = \ - handle_truncated_response(api_clients[region].describe_tags, {'FileSystemId': fs_id}, ['Tags'])['Tags'] - # Get mount targets - mount_targets = handle_truncated_response(api_clients[region].describe_mount_targets, {'FileSystemId': fs_id}, - ['MountTargets'])['MountTargets'] - file_system['mount_targets'] = {} - for mt in mount_targets: - mt_id = mt['MountTargetId'] - file_system['mount_targets'][mt_id] = mt - # Get security groups - file_system['mount_targets'][mt_id]['security_groups'] = \ - api_clients[region].describe_mount_target_security_groups(MountTargetId=mt_id)['SecurityGroups'] - self.file_systems[fs_id] = file_system - - -######################################## -# EFSConfig -######################################## - -class EFSConfig(RegionalServiceConfig): - """ - EFS configuration for all AWS regions - """ - - region_config_class = EFSRegionConfig - - def __init__(self, service_metadata, thread_config=4): - super(EFSConfig, self).__init__(service_metadata, thread_config) From 578ebfbea7b9552a8a7032e086456eea709993b5 Mon Sep 17 00:00:00 2001 From: misg Date: Wed, 6 Mar 2019 16:57:58 -0500 Subject: [PATCH 275/667] Add new API calls. --- ScoutSuite/providers/azure/facade/sqldatabase.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ScoutSuite/providers/azure/facade/sqldatabase.py b/ScoutSuite/providers/azure/facade/sqldatabase.py index 2269d5558..937585108 100644 --- a/ScoutSuite/providers/azure/facade/sqldatabase.py +++ b/ScoutSuite/providers/azure/facade/sqldatabase.py @@ -34,6 +34,16 @@ async def get_server_azure_ad_administrators(self, resource_group_name, server_n lambda: self._client.server_azure_ad_administrators.get(resource_group_name, server_name) ) + async def get_server_blob_auditing_policies(self, resource_group_name, server_name): + return await run_concurrently( + lambda: self._client.server_blob_auditing_policies.get(resource_group_name, server_name) + ) + + async def get_server_security_alert_policies(self, resource_group_name, server_name): + return await run_concurrently( + lambda: self._client.server_security_alert_policies.get(resource_group_name, server_name) + ) + async def get_servers(self): return await run_concurrently(self._client.servers.list) From 299de4942e0e952900c0acd60370a267fce04450 Mon Sep 17 00:00:00 2001 From: misg Date: Wed, 6 Mar 2019 16:58:38 -0500 Subject: [PATCH 276/667] Update fetch_all accordingly to the previously updated facade. --- .../resources/sqldatabase/server_blob_auditing_policies.py | 3 +-- .../resources/sqldatabase/server_security_alert_policies.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sqldatabase/server_blob_auditing_policies.py b/ScoutSuite/providers/azure/resources/sqldatabase/server_blob_auditing_policies.py index 7a0a5ff55..b24e1df90 100644 --- a/ScoutSuite/providers/azure/resources/sqldatabase/server_blob_auditing_policies.py +++ b/ScoutSuite/providers/azure/resources/sqldatabase/server_blob_auditing_policies.py @@ -8,9 +8,8 @@ def __init__(self, resource_group_name, server_name, facade): self.server_name = server_name self.facade = facade - # TODO: make it really async. async def fetch_all(self): - policies = self.facade.server_blob_auditing_policies.get( + policies = await self.facade.get_server_blob_auditing_policies( self.resource_group_name, self.server_name) self._parse_policies(policies) diff --git a/ScoutSuite/providers/azure/resources/sqldatabase/server_security_alert_policies.py b/ScoutSuite/providers/azure/resources/sqldatabase/server_security_alert_policies.py index 59ef96d2a..5d34faed5 100644 --- a/ScoutSuite/providers/azure/resources/sqldatabase/server_security_alert_policies.py +++ b/ScoutSuite/providers/azure/resources/sqldatabase/server_security_alert_policies.py @@ -8,9 +8,8 @@ def __init__(self, resource_group_name, server_name, facade): self.server_name = server_name self.facade = facade - # TODO: make it really async. async def fetch_all(self): - policies = self.facade.server_security_alert_policies.get( + policies = await self.facade.get_server_security_alert_policies( self.resource_group_name, self.server_name) self._parse_policies(policies) From 6213b1801df479813d5abf0c66793f4a978cd11d Mon Sep 17 00:00:00 2001 From: misg Date: Wed, 6 Mar 2019 16:59:13 -0500 Subject: [PATCH 277/667] Make server and database fetch_all really concurrent. --- .../azure/resources/sqldatabase/databases.py | 27 +++++++++++++------ .../azure/resources/sqldatabase/servers.py | 26 +++++++++++++----- 2 files changed, 38 insertions(+), 15 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sqldatabase/databases.py b/ScoutSuite/providers/azure/resources/sqldatabase/databases.py index 96b959da1..c51de1c92 100644 --- a/ScoutSuite/providers/azure/resources/sqldatabase/databases.py +++ b/ScoutSuite/providers/azure/resources/sqldatabase/databases.py @@ -1,3 +1,5 @@ +import asyncio + from ScoutSuite.providers.azure.resources.resources import AzureCompositeResources from .database_blob_auditing_policies import DatabaseBlobAuditingPolicies @@ -19,7 +21,6 @@ def __init__(self, resource_group_name, server_name, facade): self.server_name = server_name self.facade = facade - # TODO: make it really async. async def fetch_all(self): for db in await self.facade.get_databases(self.resource_group_name, self.server_name): # We do not want to scan 'master' database which is auto-generated by Azure and read-only: @@ -28,11 +29,21 @@ async def fetch_all(self): self[db.name] = { 'id': db.name, + 'name': db.name } - await self._fetch_children( - parent=self[db.name], - resource_group_name=self.resource_group_name, - server_name=self.server_name, - database_name=db.name, - facade=self.facade - ) + + # TODO: make a refactoring of the following: + if len(self) == 0: + return + tasks = { + asyncio.ensure_future( + self._fetch_children( + parent=db, + resource_group_name=self.resource_group_name, + server_name=self.server_name, + database_name=db['name'], + facade=self.facade + ) + ) for db in self.values() + } + await asyncio.wait(tasks) diff --git a/ScoutSuite/providers/azure/resources/sqldatabase/servers.py b/ScoutSuite/providers/azure/resources/sqldatabase/servers.py index b241e88d1..f2f62e807 100644 --- a/ScoutSuite/providers/azure/resources/sqldatabase/servers.py +++ b/ScoutSuite/providers/azure/resources/sqldatabase/servers.py @@ -1,3 +1,5 @@ +import asyncio + from ScoutSuite.providers.azure.resources.resources import AzureCompositeResources from ScoutSuite.providers.azure.utils import get_resource_group_name from ScoutSuite.providers.utils import get_non_provider_id @@ -17,7 +19,6 @@ class Servers(AzureCompositeResources): (ServerSecurityAlertPolicies, 'threat_detection') ] - # TODO: make it really async. async def fetch_all(self, credentials, **kwargs): # TODO: build that facade somewhere else: facade = SQLDatabaseFacade(credentials.credentials, credentials.subscription_id) @@ -29,12 +30,23 @@ async def fetch_all(self, credentials, **kwargs): self['servers'][id] = { 'id': id, - 'name': server.name + 'name': server.name, + 'resource_group_name': resource_group_name } - await self._fetch_children( - parent=self['servers'][id], - resource_group_name=resource_group_name, - server_name=server.name, - facade=facade) + + # TODO: make a refactoring of the following: + if len(self['servers']) == 0: + return + tasks = { + asyncio.ensure_future( + self._fetch_children( + parent=server, + resource_group_name=server['resource_group_name'], + server_name=server['name'], + facade=facade + ) + ) for server in self['servers'].values() + } + await asyncio.wait(tasks) self['servers_count'] = len(self['servers']) From aab240c2deb020b3ad7c6fb8dc9feb99aa3d8e29 Mon Sep 17 00:00:00 2001 From: misg Date: Wed, 6 Mar 2019 16:59:45 -0500 Subject: [PATCH 278/667] Add guards and TODOs. --- ScoutSuite/providers/aws/resources/ec2/vpcs.py | 3 +++ ScoutSuite/providers/aws/resources/regions.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/ScoutSuite/providers/aws/resources/ec2/vpcs.py b/ScoutSuite/providers/aws/resources/ec2/vpcs.py index cbc770392..b8b520059 100644 --- a/ScoutSuite/providers/aws/resources/ec2/vpcs.py +++ b/ScoutSuite/providers/aws/resources/ec2/vpcs.py @@ -19,6 +19,9 @@ async def fetch_all(self, **kwargs): name, resource = self._parse_vpc(vpc) self[name] = resource + # TODO: make a refactoring of the following: + if len(self) == 0: + return tasks = { asyncio.ensure_future( self._fetch_children( diff --git a/ScoutSuite/providers/aws/resources/regions.py b/ScoutSuite/providers/aws/resources/regions.py index 39e9f9b07..faf59865f 100644 --- a/ScoutSuite/providers/aws/resources/regions.py +++ b/ScoutSuite/providers/aws/resources/regions.py @@ -22,6 +22,9 @@ async def fetch_all(self, credentials, regions=None, partition_name='aws'): 'name': region } + # TODO: make a refactoring of the following: + if len(self['regions']) == 0: + return tasks = { asyncio.ensure_future( self._fetch_children( From 999060a1c9e0a3a24de9b4e55ffd66a3cecdd0f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sgha=C3=AFer?= Date: Thu, 7 Mar 2019 15:23:11 -0500 Subject: [PATCH 279/667] Update ScoutSuite/providers/aws/facade/cloudformation.py Co-Authored-By: Aboisier --- ScoutSuite/providers/aws/facade/cloudformation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/facade/cloudformation.py b/ScoutSuite/providers/aws/facade/cloudformation.py index 7dc4b2981..7303017f2 100644 --- a/ScoutSuite/providers/aws/facade/cloudformation.py +++ b/ScoutSuite/providers/aws/facade/cloudformation.py @@ -8,7 +8,7 @@ class CloudFormation: async def get_stacks(self, region: str): stacks = await AWSFacadeUtils.get_all_pages('cloudformation', region, 'list_stacks', 'StackSummaries') - client = await AWSFacadeUtils.get_client('cloudformation', region) + client = AWSFacadeUtils.get_client('cloudformation', region) for stack in stacks: stack_name = stack['StackName'] stack_description = await run_concurrently( From 939ae605fa2b64744888af7a3adadd85742c80cd Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Thu, 7 Mar 2019 16:09:58 -0500 Subject: [PATCH 280/667] Renamed method, removed comment --- ScoutSuite/providers/aws/resources/efs/service.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/efs/service.py b/ScoutSuite/providers/aws/resources/efs/service.py index 3917a9ecd..25390e9b0 100644 --- a/ScoutSuite/providers/aws/resources/efs/service.py +++ b/ScoutSuite/providers/aws/resources/efs/service.py @@ -7,14 +7,12 @@ class FileSystems(AWSResources): async def fetch_all(self, **kwargs): raw_file_systems = self.facade.efs.get_file_systems(self.scope['region']) for raw_file_system in raw_file_systems: - name, resource = self._parse(raw_file_system) + name, resource = self._parse_file_system(raw_file_system) self[name] = resource - def _parse(self, raw_file_system): + def _parse_file_system(self, raw_file_system): fs_id = raw_file_system.pop('FileSystemId') raw_file_system['name'] = raw_file_system.pop('Name') if 'Name' in raw_file_system else None - - # Get tags raw_file_system['tags'] = raw_file_system.pop('Tags') return fs_id, raw_file_system From 12c16862ce1d964df7a376bf0823d85eb92f3a0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sgha=C3=AFer?= Date: Thu, 7 Mar 2019 16:16:07 -0500 Subject: [PATCH 281/667] Update ScoutSuite/providers/aws/facade/efs.py Co-Authored-By: Aboisier --- ScoutSuite/providers/aws/facade/efs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/facade/efs.py b/ScoutSuite/providers/aws/facade/efs.py index 7d7a042e9..a9df5a18a 100644 --- a/ScoutSuite/providers/aws/facade/efs.py +++ b/ScoutSuite/providers/aws/facade/efs.py @@ -1,5 +1,6 @@ from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils from ScoutSuite.providers.aws.aws import handle_truncated_response +from ScoutSuite.providers.utils import run_concurrently class EFSFacade: @@ -22,4 +23,4 @@ def get_file_systems(self, region: str): security_groups = client.describe_mount_target_security_groups(MountTargetId=mount_target_id)['SecurityGroups'] file_system['MountTargets'][mount_target_id]['SecurityGroups'] = security_groups - return file_systems \ No newline at end of file + return file_systems From 871ec7541b091394d35140ead1b809fbeb545e89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sgha=C3=AFer?= Date: Thu, 7 Mar 2019 16:16:44 -0500 Subject: [PATCH 282/667] Made code async Co-Authored-By: Aboisier --- ScoutSuite/providers/aws/facade/efs.py | 14 +++++++++----- ScoutSuite/providers/aws/resources/efs/service.py | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/efs.py b/ScoutSuite/providers/aws/facade/efs.py index a9df5a18a..affaede24 100644 --- a/ScoutSuite/providers/aws/facade/efs.py +++ b/ScoutSuite/providers/aws/facade/efs.py @@ -4,23 +4,27 @@ class EFSFacade: - def get_file_systems(self, region: str): - file_systems = AWSFacadeUtils.get_all_pages('efs', region, 'describe_file_systems', 'FileSystems') + async def get_file_systems(self, region: str): + file_systems = await AWSFacadeUtils.get_all_pages('efs', region, 'describe_file_systems', 'FileSystems') client = AWSFacadeUtils.get_client('efs', region) for file_system in file_systems: file_system_id = file_system['FileSystemId'] - file_system['Tags'] = client.describe_tags(FileSystemId=file_system_id)['Tags'] + file_system['Tags'] = await run_concurrently( + lambda: client.describe_tags(FileSystemId=file_system_id)['Tags'] + ) # Get mount targets - mount_targets = AWSFacadeUtils.get_all_pages('efs', region, 'describe_mount_targets', 'MountTargets', FileSystemId=file_system_id) + mount_targets = await AWSFacadeUtils.get_all_pages('efs', region, 'describe_mount_targets', 'MountTargets', FileSystemId=file_system_id) file_system['MountTargets'] = {} for mt in mount_targets: mount_target_id = mt['MountTargetId'] file_system['MountTargets'][mount_target_id] = mt # Get security groups - security_groups = client.describe_mount_target_security_groups(MountTargetId=mount_target_id)['SecurityGroups'] + security_groups = run_concurrently( + lambda: client.describe_mount_target_security_groups(MountTargetId=mount_target_id)['SecurityGroups'] + ) file_system['MountTargets'][mount_target_id]['SecurityGroups'] = security_groups return file_systems diff --git a/ScoutSuite/providers/aws/resources/efs/service.py b/ScoutSuite/providers/aws/resources/efs/service.py index 25390e9b0..42f4092f6 100644 --- a/ScoutSuite/providers/aws/resources/efs/service.py +++ b/ScoutSuite/providers/aws/resources/efs/service.py @@ -5,7 +5,7 @@ class FileSystems(AWSResources): async def fetch_all(self, **kwargs): - raw_file_systems = self.facade.efs.get_file_systems(self.scope['region']) + raw_file_systems = await self.facade.efs.get_file_systems(self.scope['region']) for raw_file_system in raw_file_systems: name, resource = self._parse_file_system(raw_file_system) self[name] = resource From 90cb0c4b18562dd3e7039133a39ab4c3bcfa9f0b Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Fri, 8 Mar 2019 08:37:00 -0500 Subject: [PATCH 283/667] Renamed variable, ctrl+k+d --- ScoutSuite/providers/aws/facade/efs.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/efs.py b/ScoutSuite/providers/aws/facade/efs.py index affaede24..d1f1a7e75 100644 --- a/ScoutSuite/providers/aws/facade/efs.py +++ b/ScoutSuite/providers/aws/facade/efs.py @@ -10,20 +10,19 @@ async def get_file_systems(self, region: str): client = AWSFacadeUtils.get_client('efs', region) for file_system in file_systems: file_system_id = file_system['FileSystemId'] - file_system['Tags'] = await run_concurrently( - lambda: client.describe_tags(FileSystemId=file_system_id)['Tags'] - ) + file_system['Tags'] = await run_concurrently(lambda: client.describe_tags(FileSystemId=file_system_id)['Tags']) # Get mount targets mount_targets = await AWSFacadeUtils.get_all_pages('efs', region, 'describe_mount_targets', 'MountTargets', FileSystemId=file_system_id) file_system['MountTargets'] = {} - for mt in mount_targets: - mount_target_id = mt['MountTargetId'] - file_system['MountTargets'][mount_target_id] = mt + for mount_target in mount_targets: + mount_target_id = mount_target['MountTargetId'] + file_system['MountTargets'][mount_target_id] = mount_target # Get security groups security_groups = run_concurrently( - lambda: client.describe_mount_target_security_groups(MountTargetId=mount_target_id)['SecurityGroups'] + lambda: client.describe_mount_target_security_groups( + MountTargetId=mount_target_id)['SecurityGroups'] ) file_system['MountTargets'][mount_target_id]['SecurityGroups'] = security_groups From 66713a83f8f64b9e9ff9978f74100879ef186ded Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sgha=C3=AFer?= Date: Fri, 8 Mar 2019 08:38:28 -0500 Subject: [PATCH 284/667] Made code async Co-Authored-By: Aboisier --- ScoutSuite/providers/aws/facade/directconnect.py | 8 ++++++-- .../providers/aws/resources/directconnect/service.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/directconnect.py b/ScoutSuite/providers/aws/facade/directconnect.py index c238644cc..b272d51ba 100644 --- a/ScoutSuite/providers/aws/facade/directconnect.py +++ b/ScoutSuite/providers/aws/facade/directconnect.py @@ -1,6 +1,10 @@ from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils +from ScoutSuite.providers.utils import run_concurrently class DirectConnectFacade: - def get_connections(self, region): - return AWSFacadeUtils.get_client('directconnect', region).describe_connections()['connections'] \ No newline at end of file + async def get_connections(self, region): + client = AWSFacadeUtils.get_client('directconnect', region) + return await run_concurrently( + lambda: client.describe_connections()['connections'] + ) diff --git a/ScoutSuite/providers/aws/resources/directconnect/service.py b/ScoutSuite/providers/aws/resources/directconnect/service.py index 986dd9f62..97293a515 100644 --- a/ScoutSuite/providers/aws/resources/directconnect/service.py +++ b/ScoutSuite/providers/aws/resources/directconnect/service.py @@ -5,7 +5,7 @@ class Connections(AWSResources): async def fetch_all(self, **kwargs): - raw_connections = self.facade.directconnect.get_connections(self.scope['region']) + raw_connections = await self.facade.directconnect.get_connections(self.scope['region']) for raw_connection in raw_connections: name, resource = self._parse_function(raw_connection) self[name] = resource From d561d5e5a8d94ade513a791a1a2f0515cbfec45a Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Fri, 8 Mar 2019 08:41:52 -0500 Subject: [PATCH 285/667] Lint --- ScoutSuite/providers/aws/facade/directconnect.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/directconnect.py b/ScoutSuite/providers/aws/facade/directconnect.py index b272d51ba..c6a0e0b18 100644 --- a/ScoutSuite/providers/aws/facade/directconnect.py +++ b/ScoutSuite/providers/aws/facade/directconnect.py @@ -5,6 +5,4 @@ class DirectConnectFacade: async def get_connections(self, region): client = AWSFacadeUtils.get_client('directconnect', region) - return await run_concurrently( - lambda: client.describe_connections()['connections'] - ) + return await run_concurrently(lambda: client.describe_connections()['connections']) From 3753510b1ee19d459464105e4f9840af506ba332 Mon Sep 17 00:00:00 2001 From: misg Date: Fri, 8 Mar 2019 13:34:36 -0500 Subject: [PATCH 286/667] Define a facade for the Azure Storage Accounts service. --- .../providers/azure/facade/storageaccounts.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 ScoutSuite/providers/azure/facade/storageaccounts.py diff --git a/ScoutSuite/providers/azure/facade/storageaccounts.py b/ScoutSuite/providers/azure/facade/storageaccounts.py new file mode 100644 index 000000000..ee1fc19c1 --- /dev/null +++ b/ScoutSuite/providers/azure/facade/storageaccounts.py @@ -0,0 +1,15 @@ +from azure.mgmt.storage import StorageManagementClient +from ScoutSuite.providers.utils import run_concurrently + + +class StorageAccountsFacade: + def __init__(self, credentials, subscription_id): + self._client = StorageManagementClient(credentials, subscription_id) + + async def get_storage_accounts(self): + return await run_concurrently(self._client.storage_accounts.list) + + async def get_blob_containers(self, resource_group_name, storage_account_name): + return await run_concurrently( + lambda: self._client.blob_containers.list(resource_group_name, storage_account_name).value + ) From 2357af4dab3073df05e79548387942a194a5a472 Mon Sep 17 00:00:00 2001 From: misg Date: Fri, 8 Mar 2019 13:34:58 -0500 Subject: [PATCH 287/667] Define BlobContainers resources. --- .../storageaccounts/blob_containers.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 ScoutSuite/providers/azure/resources/storageaccounts/blob_containers.py diff --git a/ScoutSuite/providers/azure/resources/storageaccounts/blob_containers.py b/ScoutSuite/providers/azure/resources/storageaccounts/blob_containers.py new file mode 100644 index 000000000..8716c463b --- /dev/null +++ b/ScoutSuite/providers/azure/resources/storageaccounts/blob_containers.py @@ -0,0 +1,24 @@ +from ScoutSuite.providers.base.configs.resources import Resources + + +class BlobContainers(Resources): + + def __init__(self, resource_group_name, storage_account_name, facade): + self.resource_group_name = resource_group_name + self.storage_account_name = storage_account_name + self.facade = facade + + async def fetch_all(self): + raw_blob_containers = await self.facade.get_blob_containers( + self.resource_group_name, self.storage_account_name + ) + for raw_blob_container in raw_blob_containers: + id, blob_container = self._parse(raw_blob_container) + self[id] = blob_container + + def _parse(self, raw_blob_container): + blob_container = {} + blob_container['id'] = raw_blob_container.name + blob_container['public_access_allowed'] = raw_blob_container.public_access != "None" + + return blob_container['id'], blob_container From bf1bb0a27f7233bacc4a6cfe2efed5ac6c69d279 Mon Sep 17 00:00:00 2001 From: misg Date: Fri, 8 Mar 2019 13:35:19 -0500 Subject: [PATCH 288/667] Define StorageAccounts resources. --- .../storageaccounts/storageaccounts.py | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 ScoutSuite/providers/azure/resources/storageaccounts/storageaccounts.py diff --git a/ScoutSuite/providers/azure/resources/storageaccounts/storageaccounts.py b/ScoutSuite/providers/azure/resources/storageaccounts/storageaccounts.py new file mode 100644 index 000000000..6ccd6134b --- /dev/null +++ b/ScoutSuite/providers/azure/resources/storageaccounts/storageaccounts.py @@ -0,0 +1,55 @@ +import asyncio + +from ScoutSuite.providers.azure.resources.resources import AzureCompositeResources +from ScoutSuite.providers.azure.utils import get_resource_group_name +from ScoutSuite.providers.utils import get_non_provider_id +from ScoutSuite.providers.azure.facade.storageaccounts import StorageAccountsFacade + +from .blob_containers import BlobContainers + + +class StorageAccounts(AzureCompositeResources): + _children = [ + (BlobContainers, 'blob_containers') + ] + + async def fetch_all(self, credentials, **kwargs): + # TODO: build that facade somewhere else: + facade = StorageAccountsFacade(credentials.credentials, credentials.subscription_id) + + self['storage_accounts'] = {} + for raw_storage_account in await facade.get_storage_accounts(): + id, storage_account = self._parse(raw_storage_account) + self['storage_accounts'][id] = storage_account + + # TODO: make a refactoring of the following: + if len(self) == 0: + return + tasks = { + asyncio.ensure_future( + self._fetch_children( + parent=storage_account, + resource_group_name=storage_account['resource_group_name'], + storage_account_name=storage_account['name'], + facade=facade + ) + ) for storage_account in self['storage_accounts'].values() + } + await asyncio.wait(tasks) + + self['storage_accounts_count'] = len(self['storage_accounts']) + + def _parse(self, raw_storage_account): + storage_account = {} + raw_id = raw_storage_account.id + storage_account['id'] = get_non_provider_id(raw_id.lower()) + storage_account['resource_group_name'] = get_resource_group_name(raw_id) + storage_account['name'] = raw_storage_account.name + storage_account['https_traffic_enabled'] = raw_storage_account.enable_https_traffic_only + storage_account['public_traffic_allowed'] = self._is_public_traffic_allowed(raw_storage_account) + + return storage_account['id'], storage_account + + def _is_public_traffic_allowed(self, storage_account): + return storage_account.network_rule_set.default_action == "Allow" + From 61ebd70b92580e1997c1da8137c3831f65a70ca2 Mon Sep 17 00:00:00 2001 From: misg Date: Fri, 8 Mar 2019 14:49:20 -0500 Subject: [PATCH 289/667] Define a facade for the Azure Monitor service. --- ScoutSuite/providers/azure/facade/monitor.py | 21 ++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 ScoutSuite/providers/azure/facade/monitor.py diff --git a/ScoutSuite/providers/azure/facade/monitor.py b/ScoutSuite/providers/azure/facade/monitor.py new file mode 100644 index 000000000..c99db2344 --- /dev/null +++ b/ScoutSuite/providers/azure/facade/monitor.py @@ -0,0 +1,21 @@ +import datetime + +from azure.mgmt.monitor import MonitorManagementClient +from ScoutSuite.providers.utils import run_concurrently + + +class MonitorFacade: + def __init__(self, credentials, subscription_id): + self._client = MonitorManagementClient(credentials, subscription_id) + + async def get_activity_logs(self): + time_format = "%Y-%m-%dT%H:%M:%S.%f" + utc_now = datetime.datetime.utcnow() + end_time = utc_now.strftime(time_format) + timespan = datetime.timedelta(90) # 90 days of timespan + start_time = (utc_now - timespan).strftime(time_format) + + return await run_concurrently( + lambda: self._client.activity_logs.list( + filter="eventTimestamp ge {} and eventTimestamp le {}".format(start_time, end_time)) + ) From 5c38537142edb4ceca2708521f131ce82ba27858 Mon Sep 17 00:00:00 2001 From: misg Date: Fri, 8 Mar 2019 14:49:40 -0500 Subject: [PATCH 290/667] Define ActivityLogs resources. --- .../azure/resources/monitor/activity_logs.py | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 ScoutSuite/providers/azure/resources/monitor/activity_logs.py diff --git a/ScoutSuite/providers/azure/resources/monitor/activity_logs.py b/ScoutSuite/providers/azure/resources/monitor/activity_logs.py new file mode 100644 index 000000000..a48390b36 --- /dev/null +++ b/ScoutSuite/providers/azure/resources/monitor/activity_logs.py @@ -0,0 +1,30 @@ +from ScoutSuite.providers.base.configs.resources import Resources +from ScoutSuite.providers.azure.facade.monitor import MonitorFacade +from ScoutSuite.providers.utils import get_non_provider_id + + +class ActivityLogs(Resources): + + async def fetch_all(self, credentials, **kwargs): + # TODO: build that facade somewhere else: + facade = MonitorFacade(credentials.credentials, credentials.subscription_id) + + self['activity_logs'] = {} + self['activity_logs']['storage_accounts'] = {} + + for raw_log in await facade.get_activity_logs(): + self._parse(raw_log) + + def _parse(self, raw_log): + if raw_log.resource_type.value == 'Microsoft.Storage/storageAccounts': + self._parse_storage_account_log(raw_log) + + def _parse_storage_account_log(self, raw_log): + storage_account_id = get_non_provider_id(raw_log.resource_id.lower()) + + if storage_account_id not in self['activity_logs']['storage_accounts']: + self['activity_logs']['storage_accounts'][storage_account_id] =\ + {'access_keys_rotated': False} + + if raw_log.operation_name.value == 'Microsoft.Storage/storageAccounts/regenerateKey/action': + self['activity_logs']['storage_accounts'][storage_account_id]['access_keys_rotated'] = True From eca71b7ac3d8039bc6279e2dbeba47d02d98d2b5 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Fri, 8 Mar 2019 18:37:56 -0500 Subject: [PATCH 291/667] The deletion policy finding now also handles strings --- ScoutSuite/providers/aws/metadata.json | 2 +- .../providers/aws/services/cloudformation.py | 35 +++++++++++++++---- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/ScoutSuite/providers/aws/metadata.json b/ScoutSuite/providers/aws/metadata.json index 0b4c2b7bd..b5ee6a71c 100644 --- a/ScoutSuite/providers/aws/metadata.json +++ b/ScoutSuite/providers/aws/metadata.json @@ -43,7 +43,7 @@ } } }, - "cloudwatch": { + "cloudwatch": { "resources": { "alarms": { "api_call": "describe_alarms", diff --git a/ScoutSuite/providers/aws/services/cloudformation.py b/ScoutSuite/providers/aws/services/cloudformation.py index bef5b20f5..7e8376185 100644 --- a/ScoutSuite/providers/aws/services/cloudformation.py +++ b/ScoutSuite/providers/aws/services/cloudformation.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import json +import re from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients @@ -30,18 +31,40 @@ def parse_stack(self, global_params, region, stack): stack['termination_protection'] = stack_description['Stacks'][0]['EnableTerminationProtection'] stack['drifted'] = stack.pop('DriftInformation')['StackDriftStatus'] == 'DRIFTED' - template = api_clients[region].get_template(StackName=stack['name'])['TemplateBody']['Resources'] - stack['deletion_policy'] = 'Delete' - for group in template.keys(): - if 'DeletionPolicy' in template[group]: - stack['deletion_policy'] = template[group]['DeletionPolicy'] - break + template = api_clients[region].get_template(StackName=stack['name'])['TemplateBody'] + stack['deletion_policy'] = self.has_deletion_policy(template) stack_policy = api_clients[region].get_stack_policy(StackName=stack['name']) if 'StackPolicyBody' in stack_policy: stack['policy'] = json.loads(stack_policy['StackPolicyBody']) self.stacks[stack['name']] = stack + @staticmethod + def has_deletion_policy(template): + """ + Return region to be used for global calls such as list bucket and get bucket location + + :param template: The api response containing the stack's template + :return: + """ + has_dp = True + # If a ressource is found to not have a deletion policy or have it to delete, the boolean is switched to + # false to indicate that the ressource will be deleted once the stack is deleted + if isinstance(template, dict): + template = template['Resources'] + for group in template.keys(): + if 'DeletionPolicy' in template[group]: + if template[group]['DeletionPolicy'] is 'Delete': + has_dp = False + else: + has_dp = False + if isinstance(template, str): + if re.match(r'\"DeletionPolicy\"\s*:\s*\"Delete\"', template): + has_dp = False + elif not re.match(r'\"DeletionPolicy\"', template): + has_dp = False + return has_dp + ######################################## # CloudFormationConfig From 8360ec653286e30d3c7a234281c3314021e2e3ef Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Fri, 8 Mar 2019 18:42:52 -0500 Subject: [PATCH 292/667] Accidently changed the indentation in metadata.json --- ScoutSuite/providers/aws/metadata.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/metadata.json b/ScoutSuite/providers/aws/metadata.json index b5ee6a71c..0b4c2b7bd 100644 --- a/ScoutSuite/providers/aws/metadata.json +++ b/ScoutSuite/providers/aws/metadata.json @@ -43,7 +43,7 @@ } } }, - "cloudwatch": { + "cloudwatch": { "resources": { "alarms": { "api_call": "describe_alarms", From 41600066d251666b6bdbabd55033235bc10c0d6e Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Sat, 9 Mar 2019 15:47:19 -0500 Subject: [PATCH 293/667] Changed the favicon for each report --- ScoutSuite/output/data/html/report.html | 2 +- ScoutSuite/output/data/html/ruleset-generator.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/output/data/html/report.html b/ScoutSuite/output/data/html/report.html index e548508bb..4716f5fc6 100644 --- a/ScoutSuite/output/data/html/report.html +++ b/ScoutSuite/output/data/html/report.html @@ -10,7 +10,7 @@ Scout Suite Report - + diff --git a/ScoutSuite/output/data/html/ruleset-generator.html b/ScoutSuite/output/data/html/ruleset-generator.html index 2e9ae0226..cd9a475ab 100644 --- a/ScoutSuite/output/data/html/ruleset-generator.html +++ b/ScoutSuite/output/data/html/ruleset-generator.html @@ -10,7 +10,7 @@ Scout2 Ruleset Generator - + From 8f17a125ee300f16e9b7cf61fd6c9de5e08f8591 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Sat, 9 Mar 2019 17:37:37 -0500 Subject: [PATCH 294/667] Added rule for internet facing scheme --- .../partials/aws/services.elbv2.regions.id.vpcs.id.lbs.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/output/data/html/partials/aws/services.elbv2.regions.id.vpcs.id.lbs.html b/ScoutSuite/output/data/html/partials/aws/services.elbv2.regions.id.vpcs.id.lbs.html index ad69bfe55..05cb593d7 100644 --- a/ScoutSuite/output/data/html/partials/aws/services.elbv2.regions.id.vpcs.id.lbs.html +++ b/ScoutSuite/output/data/html/partials/aws/services.elbv2.regions.id.vpcs.id.lbs.html @@ -9,7 +9,7 @@

    Network

    • VPC: {{get_value_at 'services.elbv2.regions' region 'vpcs' vpc 'name'}} ({{vpc}})
    • DNS: {{DNSName}}
      • -
    • Scheme: {{Scheme}}
      • +
    • Scheme: {{Scheme}}
    • Type: {{Type}}
    • Availability zones:
        • From 4f65bc4ca955876429cd8d93a71b218325f9a03b Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Sat, 9 Mar 2019 17:57:21 -0500 Subject: [PATCH 295/667] Replaced base64 favicon with favicon.ico --- ScoutSuite/output/data/html/report.html | 2 +- .../output/data/html/ruleset-generator.html | 2 +- .../output/data/inc-scoutsuite/favicon.ico | Bin 0 -> 1150 bytes 3 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 ScoutSuite/output/data/inc-scoutsuite/favicon.ico diff --git a/ScoutSuite/output/data/html/report.html b/ScoutSuite/output/data/html/report.html index 4716f5fc6..05d882128 100644 --- a/ScoutSuite/output/data/html/report.html +++ b/ScoutSuite/output/data/html/report.html @@ -10,7 +10,7 @@ Scout Suite Report - + diff --git a/ScoutSuite/output/data/html/ruleset-generator.html b/ScoutSuite/output/data/html/ruleset-generator.html index cd9a475ab..61372d6e9 100644 --- a/ScoutSuite/output/data/html/ruleset-generator.html +++ b/ScoutSuite/output/data/html/ruleset-generator.html @@ -10,7 +10,7 @@ Scout2 Ruleset Generator - + diff --git a/ScoutSuite/output/data/inc-scoutsuite/favicon.ico b/ScoutSuite/output/data/inc-scoutsuite/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..2dde4a08b524bef525058ed055dd732205a2efb0 GIT binary patch literal 1150 zcmd7Qze~eF7{>88R&{c5i>>>ybf|IMz!N0&k zbka^0@du?yQEB7%NpqnTDz<`1`Q*L3+>^U&V&?HnrHpIFa>HiRX6Az-M%v>PGwx;P z%+eEK?_XS&#><8ze60M?GV>;(>^u(ffY&};Fn1NlXdxKDC7JpFIk(ma*|%Th@5r8h zlW%aZikqlvU7b~Xu3#GrI6)0=U#=>Di*w|0gc5d;MFM&UW7xzaT-Mc3#$KR(?qLfD z(7O)5K~?BkyLg7O?`TGC^`Eehb!g8+h`(XAF@iPdTpKup+lPCm_E><2IN#aOcx0BU zHLs!n@e0bO5zgt%b+@BAI@b&qF@>J%cIW@19|UGn^m?zRdu80Cm0 Date: Sat, 9 Mar 2019 23:30:23 -0500 Subject: [PATCH 296/667] Added rule checking if theres a listener with https protocol --- .../partials/aws/services.elbv2.regions.id.vpcs.id.lbs.html | 2 ++ ScoutSuite/providers/aws/services/elbv2.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/ScoutSuite/output/data/html/partials/aws/services.elbv2.regions.id.vpcs.id.lbs.html b/ScoutSuite/output/data/html/partials/aws/services.elbv2.regions.id.vpcs.id.lbs.html index 05cb593d7..f5b6ff40a 100644 --- a/ScoutSuite/output/data/html/partials/aws/services.elbv2.regions.id.vpcs.id.lbs.html +++ b/ScoutSuite/output/data/html/partials/aws/services.elbv2.regions.id.vpcs.id.lbs.html @@ -22,11 +22,13 @@

        Network

        Listeners

          + {{#each listeners}}
        • {{@key}} ({{Protocol}}{{#if SslPolicy}}, {{SslPolicy}}{{/if}})
          • {{/each}} +
        diff --git a/ScoutSuite/providers/aws/services/elbv2.py b/ScoutSuite/providers/aws/services/elbv2.py index b8f93986b..74ee21aa0 100644 --- a/ScoutSuite/providers/aws/services/elbv2.py +++ b/ScoutSuite/providers/aws/services/elbv2.py @@ -44,6 +44,7 @@ def parse_lb(self, global_params, region, lb): except Exception as e: # Network load balancers do not have security groups pass + lb['has_secure_protocol'] = False lb['listeners'] = {} # Get listeners listeners = handle_truncated_response(api_clients[region].describe_listeners, {'LoadBalancerArn': lb['arn']}, @@ -53,6 +54,9 @@ def parse_lb(self, global_params, region, lb): listener.pop('LoadBalancerArn') port = listener.pop('Port') lb['listeners'][port] = listener + if listener['Protocol'] is 'HTTPS': + lb['has_secure_protocol'] = True + # Get attributes lb['attributes'] = api_clients[region].describe_load_balancer_attributes(LoadBalancerArn=lb['arn'])[ 'Attributes'] From 2d71159e963639ca0a33abb92f82fa5c02c5dd00 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Sun, 10 Mar 2019 15:50:27 -0400 Subject: [PATCH 297/667] Removed deleted stacks --- ScoutSuite/providers/aws/facade/cloudformation.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/facade/cloudformation.py b/ScoutSuite/providers/aws/facade/cloudformation.py index 7303017f2..6ab7bd4f8 100644 --- a/ScoutSuite/providers/aws/facade/cloudformation.py +++ b/ScoutSuite/providers/aws/facade/cloudformation.py @@ -7,10 +7,11 @@ class CloudFormation: async def get_stacks(self, region: str): stacks = await AWSFacadeUtils.get_all_pages('cloudformation', region, 'list_stacks', 'StackSummaries') - + stacks = [stack for stack in stacks if not CloudFormation._is_stack_deleted(stack)] client = AWSFacadeUtils.get_client('cloudformation', region) for stack in stacks: stack_name = stack['StackName'] + stack_description = await run_concurrently( lambda: client.describe_stacks(StackName=stack_name)['Stacks'][0] ) @@ -25,3 +26,7 @@ async def get_stacks(self, region: str): stack['policy'] = json.loads(stack_policy['StackPolicyBody']) return stacks + + @staticmethod + def _is_stack_deleted(stack): + return stack.get('StackStatus', None) == 'DELETE_COMPLETE' \ No newline at end of file From dda0c97e3b829fd3be3e784afb053dca3cdfa7cb Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Sun, 10 Mar 2019 15:50:48 -0400 Subject: [PATCH 298/667] Added check for keys attribute Some stacks will return a string --- .../providers/aws/resources/cloudformation/service.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/cloudformation/service.py b/ScoutSuite/providers/aws/resources/cloudformation/service.py index 0479ef499..a7439625a 100644 --- a/ScoutSuite/providers/aws/resources/cloudformation/service.py +++ b/ScoutSuite/providers/aws/resources/cloudformation/service.py @@ -17,10 +17,12 @@ def _parse_stack(self, raw_stack): template = raw_stack.pop('template') raw_stack['deletion_policy'] = 'Delete' - for group in template.keys(): - if 'DeletionPolicy' in template[group]: - raw_stack['deletion_policy'] = template[group]['DeletionPolicy'] - break + + if hasattr(template, 'keys'): + for group in template.keys(): + if 'DeletionPolicy' in template[group]: + raw_stack['deletion_policy'] = template[group]['DeletionPolicy'] + break return raw_stack['name'], raw_stack From 5d3975f6ea8cf3628ec586139b81117fb56c09c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sgha=C3=AFer?= Date: Sun, 10 Mar 2019 16:46:12 -0400 Subject: [PATCH 299/667] Made CloudWatch facade async Co-Authored-By: Aboisier --- ScoutSuite/providers/aws/facade/cloudwatch.py | 4 ++-- ScoutSuite/providers/aws/resources/cloudwatch/service.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/cloudwatch.py b/ScoutSuite/providers/aws/facade/cloudwatch.py index 07a58e405..fbbe063c0 100644 --- a/ScoutSuite/providers/aws/facade/cloudwatch.py +++ b/ScoutSuite/providers/aws/facade/cloudwatch.py @@ -2,5 +2,5 @@ class CloudWatch: - def get_alarms(self, region): - return AWSFacadeUtils.get_all_pages('cloudwatch', region, 'describe_alarms', 'MetricAlarms') + async def get_alarms(self, region): + return await AWSFacadeUtils.get_all_pages('cloudwatch', region, 'describe_alarms', 'MetricAlarms') diff --git a/ScoutSuite/providers/aws/resources/cloudwatch/service.py b/ScoutSuite/providers/aws/resources/cloudwatch/service.py index 4b3062393..7863f75c8 100644 --- a/ScoutSuite/providers/aws/resources/cloudwatch/service.py +++ b/ScoutSuite/providers/aws/resources/cloudwatch/service.py @@ -7,7 +7,7 @@ class Alarms(AWSResources): async def fetch_all(self, **kwargs): - raw_alarms = self.facade.cloudwatch.get_alarms(self.scope['region']) + raw_alarms = await self.facade.cloudwatch.get_alarms(self.scope['region']) for raw_alarm in raw_alarms: name, resource = self._parse(raw_alarm) self[name] = resource From 8e700315bc79c3d76936b314be4dd7b46b54b42e Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Sun, 10 Mar 2019 16:47:03 -0400 Subject: [PATCH 300/667] Removed comment --- ScoutSuite/providers/aws/resources/cloudwatch/service.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/cloudwatch/service.py b/ScoutSuite/providers/aws/resources/cloudwatch/service.py index 4b3062393..bd9d4be3a 100644 --- a/ScoutSuite/providers/aws/resources/cloudwatch/service.py +++ b/ScoutSuite/providers/aws/resources/cloudwatch/service.py @@ -13,14 +13,6 @@ async def fetch_all(self, **kwargs): self[name] = resource def _parse(self, raw_alarm): - """ - Parse a single CloudWatch trail - - :param global_params: Parameters shared for all regions - :param region: Name of the AWS region - :param alarm: Alarm - """ - raw_alarm['arn'] = raw_alarm.pop('AlarmArn') raw_alarm['name'] = raw_alarm.pop('AlarmName') From a6069be7d02c418b088a197407d0189d8c1d9e43 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Sun, 10 Mar 2019 17:09:00 -0400 Subject: [PATCH 301/667] Merged resource-configs --- ...ices.cloudformation.regions.id.stacks.html | 2 +- ScoutSuite/output/data/html/report.html | 2 +- .../output/data/html/ruleset-generator.html | 2 +- .../output/data/inc-scoutsuite/favicon.ico | Bin 0 -> 1150 bytes ScoutSuite/providers/aws/configs/services.py | 10 +-- .../providers/aws/facade/cloudformation.py | 32 ++++++++++ ScoutSuite/providers/aws/facade/cloudwatch.py | 6 ++ ScoutSuite/providers/aws/facade/facade.py | 6 ++ .../aws/resources/awslambda/service.py | 1 + .../aws/resources/cloudformation/service.py | 37 +++++++++++ .../aws/resources/cloudtrail/service.py | 4 +- .../aws/resources/cloudwatch/service.py | 34 ++++++++++ .../providers/aws/services/cloudformation.py | 58 ------------------ .../providers/aws/services/cloudwatch.py | 49 --------------- 14 files changed, 126 insertions(+), 117 deletions(-) create mode 100644 ScoutSuite/output/data/inc-scoutsuite/favicon.ico create mode 100644 ScoutSuite/providers/aws/facade/cloudformation.py create mode 100644 ScoutSuite/providers/aws/facade/cloudwatch.py create mode 100644 ScoutSuite/providers/aws/resources/cloudformation/service.py create mode 100644 ScoutSuite/providers/aws/resources/cloudwatch/service.py delete mode 100644 ScoutSuite/providers/aws/services/cloudformation.py delete mode 100644 ScoutSuite/providers/aws/services/cloudwatch.py diff --git a/ScoutSuite/output/data/html/partials/aws/services.cloudformation.regions.id.stacks.html b/ScoutSuite/output/data/html/partials/aws/services.cloudformation.regions.id.stacks.html index 93d540637..258c5ca3f 100644 --- a/ScoutSuite/output/data/html/partials/aws/services.cloudformation.regions.id.stacks.html +++ b/ScoutSuite/output/data/html/partials/aws/services.cloudformation.regions.id.stacks.html @@ -19,7 +19,7 @@

        Information

        None {{/if}}
        -
        Termination protection enabled: {{termination_protection}}
        +
        Termination protection enabled: {{EnableTerminationProtection}}
        Configuration has drifted: {{drifted}}
        Deletion policy: {{deletion_policy}}
    diff --git a/ScoutSuite/output/data/html/report.html b/ScoutSuite/output/data/html/report.html index e548508bb..05d882128 100644 --- a/ScoutSuite/output/data/html/report.html +++ b/ScoutSuite/output/data/html/report.html @@ -10,7 +10,7 @@ Scout Suite Report - + diff --git a/ScoutSuite/output/data/html/ruleset-generator.html b/ScoutSuite/output/data/html/ruleset-generator.html index 2e9ae0226..61372d6e9 100644 --- a/ScoutSuite/output/data/html/ruleset-generator.html +++ b/ScoutSuite/output/data/html/ruleset-generator.html @@ -10,7 +10,7 @@ Scout2 Ruleset Generator - + diff --git a/ScoutSuite/output/data/inc-scoutsuite/favicon.ico b/ScoutSuite/output/data/inc-scoutsuite/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..2dde4a08b524bef525058ed055dd732205a2efb0 GIT binary patch literal 1150 zcmd7Qze~eF7{>88R&{c5i>>>ybf|IMz!N0&k zbka^0@du?yQEB7%NpqnTDz<`1`Q*L3+>^U&V&?HnrHpIFa>HiRX6Az-M%v>PGwx;P z%+eEK?_XS&#><8ze60M?GV>;(>^u(ffY&};Fn1NlXdxKDC7JpFIk(ma*|%Th@5r8h zlW%aZikqlvU7b~Xu3#GrI6)0=U#=>Di*w|0gc5d;MFM&UW7xzaT-Mc3#$KR(?qLfD z(7O)5K~?BkyLg7O?`TGC^`Eehb!g8+h`(XAF@iPdTpKup+lPCm_E><2IN#aOcx0BU zHLs!n@e0bO5zgt%b+@BAI@b&qF@>J%cIW@19|UGn^m?zRdu80Cm0 Date: Sun, 10 Mar 2019 17:25:47 -0400 Subject: [PATCH 302/667] Fixed pre processing being called when EC2 is not enabled --- ScoutSuite/providers/aws/provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index 5814326d9..28dbac042 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -59,13 +59,13 @@ def preprocessing(self, ip_ranges=None, ip_ranges_name_key=None): :return: None """ ip_ranges = [] if ip_ranges is None else ip_ranges - self._map_all_sgs() self._map_all_subnets() self._set_emr_vpc_ids() # self.parse_elb_policies() # Various data processing calls if 'ec2' in self.service_list: + self._map_all_sgs() self._check_ec2_zone_distribution() self._add_security_group_name_to_ec2_grants() self._add_last_snapshot_date_to_ec2_volumes() From eec2674af1ddaf76168b97a1f5863d541c30ef71 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Sun, 10 Mar 2019 17:26:01 -0400 Subject: [PATCH 303/667] Fixed processing callbacks being called when services are disabled --- ScoutSuite/providers/base/provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/base/provider.py b/ScoutSuite/providers/base/provider.py index 481b14ceb..0b9e13b2f 100644 --- a/ScoutSuite/providers/base/provider.py +++ b/ScoutSuite/providers/base/provider.py @@ -213,7 +213,7 @@ def _process_metadata_callbacks(self): # Service-level summaries for service_group in self.metadata: for service in self.metadata[service_group]: - if service == 'summaries': + if service == 'summaries' or service not in self.service_list: continue # Reset external attack surface if 'summaries' in self.metadata[service_group][service]: From 25e2ed71c8ea35519498425f96006ad0569124b0 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Sun, 10 Mar 2019 18:32:55 -0400 Subject: [PATCH 304/667] Made parse method names more specific --- ScoutSuite/providers/aws/resources/cloudtrail/service.py | 4 ++-- ScoutSuite/providers/aws/resources/cloudwatch/service.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/cloudtrail/service.py b/ScoutSuite/providers/aws/resources/cloudtrail/service.py index b14f90b13..a2a962c12 100644 --- a/ScoutSuite/providers/aws/resources/cloudtrail/service.py +++ b/ScoutSuite/providers/aws/resources/cloudtrail/service.py @@ -9,10 +9,10 @@ class Trails(AWSResources): async def fetch_all(self, **kwargs): raw_trails = await self.facade.cloudtrail.get_trails(self.scope['region']) for raw_trail in raw_trails: - name, resource = self._parse(raw_trail) + name, resource = self._parse_trail(raw_trail) self[name] = resource - def _parse(self, raw_trail): + def _parse_trail(self, raw_trail): trail = {'name': raw_trail.pop('Name')} trail_id = get_non_provider_id(trail['name']) diff --git a/ScoutSuite/providers/aws/resources/cloudwatch/service.py b/ScoutSuite/providers/aws/resources/cloudwatch/service.py index 32b720626..29454e18f 100644 --- a/ScoutSuite/providers/aws/resources/cloudwatch/service.py +++ b/ScoutSuite/providers/aws/resources/cloudwatch/service.py @@ -9,10 +9,10 @@ class Alarms(AWSResources): async def fetch_all(self, **kwargs): raw_alarms = await self.facade.cloudwatch.get_alarms(self.scope['region']) for raw_alarm in raw_alarms: - name, resource = self._parse(raw_alarm) + name, resource = self._parse_alarm(raw_alarm) self[name] = resource - def _parse(self, raw_alarm): + def _parse_alarm(self, raw_alarm): raw_alarm['arn'] = raw_alarm.pop('AlarmArn') raw_alarm['name'] = raw_alarm.pop('AlarmName') From 00e11ad19c05142f8eedd467ffaf0e6a8e504198 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Sun, 10 Mar 2019 19:40:40 -0400 Subject: [PATCH 305/667] Stuck at dealing with unhashable dicts and bad indices in provider.py, I still don't get python --- ScoutSuite/providers/aws/provider.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index a2f3412aa..5ea7ad86f 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -67,6 +67,7 @@ def preprocessing(self, ip_ranges=None, ip_ranges_name_key=None): # Various data processing calls self._check_ec2_zone_distribution() self._add_security_group_name_to_ec2_grants() + self._add_security_group_data_to_elbv2() self._add_last_snapshot_date_to_ec2_volumes() self._process_cloudtrail_trails(self.services['cloudtrail']) self._add_cidr_display_name(ip_ranges, ip_ranges_name_key) @@ -104,6 +105,20 @@ def _add_security_group_name_to_ec2_grants(self): self.add_security_group_name_to_ec2_grants_callback, {'AWSAccountId': self.aws_account_id}) + def _add_security_group_data_to_elbv2(self): + ec2_config = self.services['ec2'] + elbv2_config = self.services['elbv2'] + for region in elbv2_config['regions']: + for vpc in elbv2_config['regions'][region]['vpcs']: + for lb in elbv2_config['regions'][region]['vpcs'][vpc]['lbs']: + security_groups = ec2_config['regions'][region]['vpcs'][vpc]['security_groups'] + for i in range(0, len(elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'])): + for security_group in security_groups: + if elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][i]['GroupId'] \ + is security_group['id']: + elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][i] = \ + security_group + def _check_ec2_zone_distribution(self): regions = self.services['ec2']['regions'].values() self.services['ec2']['number_of_regions_with_instances'] = sum(r['instances_count'] > 0 for r in regions) @@ -597,7 +612,7 @@ def get_db_attack_surface(self, current_config, path, current_path, db_id, callb service_config = self.services[service] manage_dictionary(service_config, 'external_attack_surface', {}) if (service == 'redshift' or service == 'rds') and 'PubliclyAccessible' in current_config and current_config[ - 'PubliclyAccessible']: + 'PubliclyAccessible']: public_dns = current_config['Endpoint']['Address'] listeners = [current_config['Endpoint']['Port']] security_groups = current_config['VpcSecurityGroups'] From 59c09389e17a253aca014d90e807cc53cc2c3c4b Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 11 Mar 2019 00:28:21 -0400 Subject: [PATCH 306/667] Define a facade for the Azure Key Vault service. --- ScoutSuite/providers/azure/facade/keyvault.py | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 ScoutSuite/providers/azure/facade/keyvault.py diff --git a/ScoutSuite/providers/azure/facade/keyvault.py b/ScoutSuite/providers/azure/facade/keyvault.py new file mode 100644 index 000000000..da55faabf --- /dev/null +++ b/ScoutSuite/providers/azure/facade/keyvault.py @@ -0,0 +1,10 @@ +from azure.mgmt.keyvault import KeyVaultManagementClient +from ScoutSuite.providers.utils import run_concurrently + + +class KeyVaultFacade: + def __init__(self, credentials, subscription_id): + self._client = KeyVaultManagementClient(credentials, subscription_id) + + async def get_key_vaults(self): + return await run_concurrently(self._client.vaults.list_by_subscription) From 2922195fd589c62dc56dbc762d75cc931b887b6e Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 11 Mar 2019 00:29:50 -0400 Subject: [PATCH 307/667] Define KeyVaults resources. --- .../azure/resources/keyvault/key_vaults.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 ScoutSuite/providers/azure/resources/keyvault/key_vaults.py diff --git a/ScoutSuite/providers/azure/resources/keyvault/key_vaults.py b/ScoutSuite/providers/azure/resources/keyvault/key_vaults.py new file mode 100644 index 000000000..39f843005 --- /dev/null +++ b/ScoutSuite/providers/azure/resources/keyvault/key_vaults.py @@ -0,0 +1,28 @@ +from ScoutSuite.providers.base.configs.resources import Resources +from ScoutSuite.providers.azure.facade.keyvault import KeyVaultFacade +from ScoutSuite.providers.utils import get_non_provider_id + + +class KeyVaults(Resources): + + async def fetch_all(self, credentials, **kwargs): + # TODO: build that facade somewhere else: + facade = KeyVaultFacade(credentials.credentials, credentials.subscription_id) + + self['vaults'] = {} + for raw_vault in await facade.get_key_vaults(): + id, vault = self._parse(raw_vault) + self['vaults'][id] = vault + + self['vaults_count'] = len(self['vaults']) + + def _parse(self, raw_vault): + vault = {} + vault['id'] = get_non_provider_id(raw_vault.id) + vault['name'] = raw_vault.name + vault['public_access_allowed'] = self._is_public_access_allowed(raw_vault) + + return vault['id'], vault + + def _is_public_access_allowed(self, raw_vault): + return raw_vault.properties.network_acls is None From 2111d44343ed9152d87dda0eb0e426f942d3d695 Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Mon, 11 Mar 2019 10:38:08 +0100 Subject: [PATCH 308/667] Partial implementation of EC2 user data secrets identification --- .../providers/aws/resources/ec2/instances.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/ScoutSuite/providers/aws/resources/ec2/instances.py b/ScoutSuite/providers/aws/resources/ec2/instances.py index f173cd63f..6e22f19c8 100644 --- a/ScoutSuite/providers/aws/resources/ec2/instances.py +++ b/ScoutSuite/providers/aws/resources/ec2/instances.py @@ -1,6 +1,7 @@ from ScoutSuite.providers.aws.resources.resources import AWSResources from ScoutSuite.providers.aws.aws import get_name from ScoutSuite.providers.aws.utils import get_keys +import re class EC2Instances(AWSResources): @@ -17,6 +18,7 @@ async def _parse_instance(self, raw_instance): instance['reservation_id'] = raw_instance['ReservationId'] instance['monitoring_enabled'] = raw_instance['Monitoring']['State'] == 'enabled' instance['user_data'] = await self.facade.ec2.get_instance_user_data(self.scope['region'], id) + instance['user_data_secrets'] = self._identify_user_data_secrets(instance['user_data']) get_name(raw_instance, instance, 'InstanceId') get_keys(raw_instance, instance, ['KeyName', 'LaunchTime', 'InstanceType', 'State', 'IamInstanceProfile', 'SubnetId']) @@ -28,3 +30,24 @@ async def _parse_instance(self, raw_instance): instance['network_interfaces'][eni['NetworkInterfaceId']] = nic return id, instance + + @staticmethod + def _identify_user_data_secrets(user_data): + """ + Parses EC2 user data in order to identify secrets. + """ + secrets = {} + + if user_data: + aws_access_key_regex = re.compile('AKIA[0-9A-Z]{16}') + aws_secret_access_key_regex = re.compile('[0-9a-zA-Z/+]{40}') + rsa_private_key_regex = re.compile('(-----(\bBEGIN\b|\bEND\b) ((\bRSA PRIVATE KEY\b)|(\bCERTIFICATE\b))-----)') + + if aws_access_key_regex.search(user_data): + secrets['aws_access_key']: aws_access_key_regex.search(user_data) + if aws_secret_access_key_regex.search(user_data): + secrets['aws_secret_access_key']: aws_secret_access_key_regex.search(user_data) + if rsa_private_key_regex.search(user_data): + secrets['rsa_private_key']: rsa_private_key_regex.search(user_data) + + return secrets From 1768aa9aeddb35c370e74da8cbf8ef2fea396a74 Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Mon, 11 Mar 2019 11:13:09 +0100 Subject: [PATCH 309/667] Partial implementation of EC2 user data secrets identification --- .../providers/aws/resources/ec2/instances.py | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/ec2/instances.py b/ScoutSuite/providers/aws/resources/ec2/instances.py index 6e22f19c8..4ea524224 100644 --- a/ScoutSuite/providers/aws/resources/ec2/instances.py +++ b/ScoutSuite/providers/aws/resources/ec2/instances.py @@ -34,7 +34,7 @@ async def _parse_instance(self, raw_instance): @staticmethod def _identify_user_data_secrets(user_data): """ - Parses EC2 user data in order to identify secrets. + Parses EC2 user data in order to identify secrets and credentials.. """ secrets = {} @@ -42,12 +42,22 @@ def _identify_user_data_secrets(user_data): aws_access_key_regex = re.compile('AKIA[0-9A-Z]{16}') aws_secret_access_key_regex = re.compile('[0-9a-zA-Z/+]{40}') rsa_private_key_regex = re.compile('(-----(\bBEGIN\b|\bEND\b) ((\bRSA PRIVATE KEY\b)|(\bCERTIFICATE\b))-----)') - - if aws_access_key_regex.search(user_data): - secrets['aws_access_key']: aws_access_key_regex.search(user_data) - if aws_secret_access_key_regex.search(user_data): - secrets['aws_secret_access_key']: aws_secret_access_key_regex.search(user_data) - if rsa_private_key_regex.search(user_data): - secrets['rsa_private_key']: rsa_private_key_regex.search(user_data) + keywords = ['password', 'secret'] + + aws_access_key_list = aws_access_key_regex.findall(user_data) + if aws_access_key_list: + secrets['aws_access_key'] = aws_access_key_list + aws_secret_access_key_list = aws_secret_access_key_regex.findall(user_data) + if aws_secret_access_key_list: + secrets['aws_secret_access_key'] = aws_secret_access_key_list + rsa_private_key_list = rsa_private_key_regex.findall(user_data) + if rsa_private_key_list: + secrets['rsa_private_key'] = rsa_private_key_list + word_list = [] + for word in keywords: + if word in user_data.lower(): + word_list.append(word) + if word_list: + secrets['word'] = word_list return secrets From 273bec43ddedb37c5b07de9a69a083c8a8edac9c Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Mon, 11 Mar 2019 12:06:51 +0100 Subject: [PATCH 310/667] Implementation of EC2 user data secrets identification --- ScoutSuite/providers/aws/resources/ec2/instances.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/ec2/instances.py b/ScoutSuite/providers/aws/resources/ec2/instances.py index 4ea524224..ba65a4164 100644 --- a/ScoutSuite/providers/aws/resources/ec2/instances.py +++ b/ScoutSuite/providers/aws/resources/ec2/instances.py @@ -41,23 +41,23 @@ def _identify_user_data_secrets(user_data): if user_data: aws_access_key_regex = re.compile('AKIA[0-9A-Z]{16}') aws_secret_access_key_regex = re.compile('[0-9a-zA-Z/+]{40}') - rsa_private_key_regex = re.compile('(-----(\bBEGIN\b|\bEND\b) ((\bRSA PRIVATE KEY\b)|(\bCERTIFICATE\b))-----)') - keywords = ['password', 'secret'] + rsa_private_key_regex = re.compile('(-----BEGIN RSA PRIVATE KEY-----(?s).+?-----END .+?-----)') + keywords = ['password', 'secret', 'aws_access_key_id', 'aws_secret_access_key', 'aws_session_token'] aws_access_key_list = aws_access_key_regex.findall(user_data) if aws_access_key_list: - secrets['aws_access_key'] = aws_access_key_list + secrets['AWS Access Key IDs'] = aws_access_key_list aws_secret_access_key_list = aws_secret_access_key_regex.findall(user_data) if aws_secret_access_key_list: - secrets['aws_secret_access_key'] = aws_secret_access_key_list + secrets['AWS Secret Access Keys'] = aws_secret_access_key_list rsa_private_key_list = rsa_private_key_regex.findall(user_data) if rsa_private_key_list: - secrets['rsa_private_key'] = rsa_private_key_list + secrets['Private Keys'] = rsa_private_key_list word_list = [] for word in keywords: if word in user_data.lower(): word_list.append(word) if word_list: - secrets['word'] = word_list + secrets['Flagged Words'] = word_list return secrets From 01e09dbb7da6966b52e051d8cc85749e7a14e27a Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Mon, 11 Mar 2019 12:09:20 +0100 Subject: [PATCH 311/667] Update view for EC2 user data secrets --- ...rvices.ec2.regions.id.vpcs.id.instances.html | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/output/data/html/partials/aws/services.ec2.regions.id.vpcs.id.instances.html b/ScoutSuite/output/data/html/partials/aws/services.ec2.regions.id.vpcs.id.instances.html index c176cb407..d0de73a74 100644 --- a/ScoutSuite/output/data/html/partials/aws/services.ec2.regions.id.vpcs.id.instances.html +++ b/ScoutSuite/output/data/html/partials/aws/services.ec2.regions.id.vpcs.id.instances.html @@ -33,8 +33,23 @@

    Network interfaces

    User data

    - 
    {{#each (split_lines user_data)}}   {{this}}
    {{/each}}
    + {{#each (split_lines user_data)}} {{this}}
    {{/each}}
    + {{#if user_data_secrets}} +
    Potential Secrets
    +
      + {{#each user_data_secrets}} +
    • + {{@key}} +
        + {{#each this}} +
      • {{this}}
        • + {{/each}} +
      +
      • + {{/each}} +
    + {{/if}}
    {{/if}} From e837b06043c52c5d7cbe17886a3a75cce298420d Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Mon, 11 Mar 2019 12:09:31 +0100 Subject: [PATCH 312/667] Include finding for EC2 user data secrets --- .../aws/rules/findings/ec2-instance-userdata.json | 10 ++++++++++ ScoutSuite/providers/aws/rules/rulesets/default.json | 6 ++++++ 2 files changed, 16 insertions(+) create mode 100644 ScoutSuite/providers/aws/rules/findings/ec2-instance-userdata.json diff --git a/ScoutSuite/providers/aws/rules/findings/ec2-instance-userdata.json b/ScoutSuite/providers/aws/rules/findings/ec2-instance-userdata.json new file mode 100644 index 000000000..0d8ae9886 --- /dev/null +++ b/ScoutSuite/providers/aws/rules/findings/ec2-instance-userdata.json @@ -0,0 +1,10 @@ +{ + "description": "Secrets in instance user data (potential)", + "rationale": "It was detected that the EC2 instance was configured with user data, which could potentially include secrets. Although user data can only be accessed from within the instance itself, the data is not protected by cryptographic methods. Anyone who can access the instance can view its metadata. It should therefore be ensured that sensitive data, such as passwords and SSH keys, are not stored as user data.", + "path": "ec2.regions.id.vpcs.id.instances.id", + "dashboard_name": "Instances", + "conditions": [ "and", + [ "ec2.regions.id.vpcs.id.instances.id.user_data_secrets", "notEmpty", "" ] + ], + "id_suffix": "potential_secrets" +} diff --git a/ScoutSuite/providers/aws/rules/rulesets/default.json b/ScoutSuite/providers/aws/rules/rulesets/default.json index b66077659..32daf7a09 100644 --- a/ScoutSuite/providers/aws/rules/rulesets/default.json +++ b/ScoutSuite/providers/aws/rules/rulesets/default.json @@ -283,6 +283,12 @@ "level": "warning" } ], + "ec2-instance-userdata.json": [ + { + "enabled": true, + "level": "danger" + } + ], "elb-no-access-logs.json": [ { "enabled": true, From fe0b953c22296678bc8c9903c4207d127dff33fa Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Mon, 11 Mar 2019 13:00:23 +0100 Subject: [PATCH 313/667] Rename finding --- ...e-userdata.json => ec2-instance-with-user-data-secrets.json} | 0 ScoutSuite/providers/aws/rules/rulesets/default.json | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename ScoutSuite/providers/aws/rules/findings/{ec2-instance-userdata.json => ec2-instance-with-user-data-secrets.json} (100%) diff --git a/ScoutSuite/providers/aws/rules/findings/ec2-instance-userdata.json b/ScoutSuite/providers/aws/rules/findings/ec2-instance-with-user-data-secrets.json similarity index 100% rename from ScoutSuite/providers/aws/rules/findings/ec2-instance-userdata.json rename to ScoutSuite/providers/aws/rules/findings/ec2-instance-with-user-data-secrets.json diff --git a/ScoutSuite/providers/aws/rules/rulesets/default.json b/ScoutSuite/providers/aws/rules/rulesets/default.json index 32daf7a09..2fd25876a 100644 --- a/ScoutSuite/providers/aws/rules/rulesets/default.json +++ b/ScoutSuite/providers/aws/rules/rulesets/default.json @@ -283,7 +283,7 @@ "level": "warning" } ], - "ec2-instance-userdata.json": [ + "ec2-instance-with-user-data-secrets.json": [ { "enabled": true, "level": "danger" From 3cce8a4deb5343a5d6dccf7fbec15aacf1b490c8 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 11 Mar 2019 10:16:11 -0400 Subject: [PATCH 314/667] Integrated parts of connect_service to get_client --- ScoutSuite/providers/aws/facade/awslambda.py | 6 +-- ScoutSuite/providers/aws/facade/basefacade.py | 7 +++ .../providers/aws/facade/cloudformation.py | 7 +-- ScoutSuite/providers/aws/facade/cloudtrail.py | 5 +- ScoutSuite/providers/aws/facade/cloudwatch.py | 5 +- .../providers/aws/facade/directconnect.py | 5 +- ScoutSuite/providers/aws/facade/ec2.py | 19 ++++---- ScoutSuite/providers/aws/facade/efs.py | 9 ++-- ScoutSuite/providers/aws/facade/facade.py | 46 ++++++++++++++----- ScoutSuite/providers/aws/facade/utils.py | 18 ++++++-- ScoutSuite/providers/aws/resources/regions.py | 3 ++ .../providers/aws/resources/resources.py | 6 +-- 12 files changed, 93 insertions(+), 43 deletions(-) create mode 100644 ScoutSuite/providers/aws/facade/basefacade.py diff --git a/ScoutSuite/providers/aws/facade/awslambda.py b/ScoutSuite/providers/aws/facade/awslambda.py index ed5d51262..893b67cbd 100644 --- a/ScoutSuite/providers/aws/facade/awslambda.py +++ b/ScoutSuite/providers/aws/facade/awslambda.py @@ -1,6 +1,6 @@ from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils +from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade - -class LambdaFacade: +class LambdaFacade(AWSBaseFacade): async def get_functions(self, region): - return await AWSFacadeUtils.get_all_pages('lambda', region, 'list_functions', 'Functions') + return await AWSFacadeUtils.get_all_pages('lambda', region, self.session, 'list_functions', 'Functions') diff --git a/ScoutSuite/providers/aws/facade/basefacade.py b/ScoutSuite/providers/aws/facade/basefacade.py new file mode 100644 index 000000000..0725be995 --- /dev/null +++ b/ScoutSuite/providers/aws/facade/basefacade.py @@ -0,0 +1,7 @@ +import boto3 + + +class AWSBaseFacade(object): + def __init__(self, session: boto3.session.Session): + self.session = session + \ No newline at end of file diff --git a/ScoutSuite/providers/aws/facade/cloudformation.py b/ScoutSuite/providers/aws/facade/cloudformation.py index 6ab7bd4f8..99408ca3d 100644 --- a/ScoutSuite/providers/aws/facade/cloudformation.py +++ b/ScoutSuite/providers/aws/facade/cloudformation.py @@ -2,13 +2,14 @@ from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils from ScoutSuite.providers.utils import run_concurrently +from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade -class CloudFormation: +class CloudFormation(AWSBaseFacade): async def get_stacks(self, region: str): - stacks = await AWSFacadeUtils.get_all_pages('cloudformation', region, 'list_stacks', 'StackSummaries') + stacks = await AWSFacadeUtils.get_all_pages('cloudformation', region, self.session, 'list_stacks', 'StackSummaries') stacks = [stack for stack in stacks if not CloudFormation._is_stack_deleted(stack)] - client = AWSFacadeUtils.get_client('cloudformation', region) + client = AWSFacadeUtils.get_client('cloudformation', region, self.session) for stack in stacks: stack_name = stack['StackName'] diff --git a/ScoutSuite/providers/aws/facade/cloudtrail.py b/ScoutSuite/providers/aws/facade/cloudtrail.py index 58bafc2c3..f34392ba5 100644 --- a/ScoutSuite/providers/aws/facade/cloudtrail.py +++ b/ScoutSuite/providers/aws/facade/cloudtrail.py @@ -1,10 +1,11 @@ from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils from ScoutSuite.providers.utils import run_concurrently +from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade -class CloudTrailFacade: +class CloudTrailFacade(AWSBaseFacade): async def get_trails(self, region): - client = AWSFacadeUtils.get_client('cloudtrail', region) + client = AWSFacadeUtils.get_client('cloudtrail', region, self.session) trails = await run_concurrently( lambda: client.describe_trails()['trailList'] ) diff --git a/ScoutSuite/providers/aws/facade/cloudwatch.py b/ScoutSuite/providers/aws/facade/cloudwatch.py index fbbe063c0..7403388f6 100644 --- a/ScoutSuite/providers/aws/facade/cloudwatch.py +++ b/ScoutSuite/providers/aws/facade/cloudwatch.py @@ -1,6 +1,7 @@ from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils +from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade -class CloudWatch: +class CloudWatch(AWSBaseFacade): async def get_alarms(self, region): - return await AWSFacadeUtils.get_all_pages('cloudwatch', region, 'describe_alarms', 'MetricAlarms') + return await AWSFacadeUtils.get_all_pages('cloudwatch', region, self.session, 'describe_alarms', 'MetricAlarms') diff --git a/ScoutSuite/providers/aws/facade/directconnect.py b/ScoutSuite/providers/aws/facade/directconnect.py index c6a0e0b18..df24839cb 100644 --- a/ScoutSuite/providers/aws/facade/directconnect.py +++ b/ScoutSuite/providers/aws/facade/directconnect.py @@ -1,8 +1,9 @@ from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils from ScoutSuite.providers.utils import run_concurrently +from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade -class DirectConnectFacade: +class DirectConnectFacade(AWSBaseFacade): async def get_connections(self, region): - client = AWSFacadeUtils.get_client('directconnect', region) + client = AWSFacadeUtils.get_client('directconnect', region, self.session) return await run_concurrently(lambda: client.describe_connections()['connections']) diff --git a/ScoutSuite/providers/aws/facade/ec2.py b/ScoutSuite/providers/aws/facade/ec2.py index e9939f601..704b61358 100644 --- a/ScoutSuite/providers/aws/facade/ec2.py +++ b/ScoutSuite/providers/aws/facade/ec2.py @@ -3,11 +3,12 @@ from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils from ScoutSuite.providers.utils import run_concurrently +from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade -class EC2Facade: +class EC2Facade(AWSBaseFacade): async def get_instance_user_data(self, region: str, instance_id: str): - ec2_client = AWSFacadeUtils.get_client('ec2', region) + ec2_client = AWSFacadeUtils.get_client('ec2', region, self.session) user_data_response = await run_concurrently( lambda: ec2_client.describe_instance_attribute(Attribute='userData', InstanceId=instance_id)) @@ -19,7 +20,7 @@ async def get_instance_user_data(self, region: str, instance_id: str): async def get_instances(self, region, vpc): filters = [{'Name': 'vpc-id', 'Values': [vpc]}] reservations =\ - await AWSFacadeUtils.get_all_pages('ec2', region, 'describe_instances', 'Reservations', Filters=filters) + await AWSFacadeUtils.get_all_pages('ec2', region, self.session, 'describe_instances', 'Reservations', Filters=filters) instances = [] for reservation in reservations: @@ -32,7 +33,7 @@ async def get_instances(self, region, vpc): async def get_security_groups(self, region, vpc): filters = [{'Name': 'vpc-id', 'Values': [vpc]}] return await AWSFacadeUtils.get_all_pages( - 'ec2', region, 'describe_security_groups', 'SecurityGroups', Filters=filters) + 'ec2', region, self.session, 'describe_security_groups', 'SecurityGroups', Filters=filters) async def get_vpcs(self, region): ec2_client = await run_concurrently(lambda: boto3.client('ec2', region_name=region)) @@ -40,7 +41,7 @@ async def get_vpcs(self, region): async def get_images(self, region, owner_id): filters = [{'Name': 'owner-id', 'Values': [owner_id]}] - client = AWSFacadeUtils.get_client('ec2', region) + client = AWSFacadeUtils.get_client('ec2', region, self.session) response = await run_concurrently(lambda: client.describe_images(Filters=filters)) return response['Images'] @@ -48,17 +49,17 @@ async def get_images(self, region, owner_id): async def get_network_interfaces(self, region, vpc): filters = [{'Name': 'vpc-id', 'Values': [vpc]}] return await AWSFacadeUtils.get_all_pages( - 'ec2', region, 'describe_network_interfaces', 'NetworkInterfaces', Filters=filters) + 'ec2', region, self.session, 'describe_network_interfaces', 'NetworkInterfaces', Filters=filters) async def get_volumes(self, region): - return await AWSFacadeUtils.get_all_pages('ec2', region, 'describe_volumes', 'Volumes') + return await AWSFacadeUtils.get_all_pages('ec2', region, self.session, 'describe_volumes', 'Volumes') async def get_snapshots(self, region, owner_id): filters = [{'Name': 'owner-id', 'Values': [owner_id]}] snapshots = await AWSFacadeUtils.get_all_pages( - 'ec2', region, 'describe_snapshots', 'Snapshots', Filters=filters) + 'ec2', region, self.session, 'describe_snapshots', 'Snapshots', Filters=filters) - ec2_client = AWSFacadeUtils.get_client('ec2', region) + ec2_client = AWSFacadeUtils.get_client('ec2', region, self.session) for snapshot in snapshots: snapshot['CreateVolumePermissions'] = await run_concurrently(lambda: ec2_client.describe_snapshot_attribute( Attribute='createVolumePermission', diff --git a/ScoutSuite/providers/aws/facade/efs.py b/ScoutSuite/providers/aws/facade/efs.py index d1f1a7e75..b33955d89 100644 --- a/ScoutSuite/providers/aws/facade/efs.py +++ b/ScoutSuite/providers/aws/facade/efs.py @@ -1,19 +1,20 @@ from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils from ScoutSuite.providers.aws.aws import handle_truncated_response from ScoutSuite.providers.utils import run_concurrently +from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade -class EFSFacade: +class EFSFacade(AWSBaseFacade): async def get_file_systems(self, region: str): - file_systems = await AWSFacadeUtils.get_all_pages('efs', region, 'describe_file_systems', 'FileSystems') + file_systems = await AWSFacadeUtils.get_all_pages('efs', region, self.session, 'describe_file_systems', 'FileSystems') - client = AWSFacadeUtils.get_client('efs', region) + client = AWSFacadeUtils.get_client('efs', region, self.session) for file_system in file_systems: file_system_id = file_system['FileSystemId'] file_system['Tags'] = await run_concurrently(lambda: client.describe_tags(FileSystemId=file_system_id)['Tags']) # Get mount targets - mount_targets = await AWSFacadeUtils.get_all_pages('efs', region, 'describe_mount_targets', 'MountTargets', FileSystemId=file_system_id) + mount_targets = await AWSFacadeUtils.get_all_pages('efs', region, self.session, 'describe_mount_targets', 'MountTargets', FileSystemId=file_system_id) file_system['MountTargets'] = {} for mount_target in mount_targets: mount_target_id = mount_target['MountTargetId'] diff --git a/ScoutSuite/providers/aws/facade/facade.py b/ScoutSuite/providers/aws/facade/facade.py index fa348ade2..430909313 100644 --- a/ScoutSuite/providers/aws/facade/facade.py +++ b/ScoutSuite/providers/aws/facade/facade.py @@ -1,5 +1,6 @@ from collections import Counter from botocore.session import Session +import boto3 from ScoutSuite.providers.aws.facade.awslambda import LambdaFacade from ScoutSuite.providers.aws.facade.cloudformation import CloudFormation @@ -8,20 +9,21 @@ from ScoutSuite.providers.aws.facade.ec2 import EC2Facade from ScoutSuite.providers.aws.facade.efs import EFSFacade from ScoutSuite.providers.aws.facade.directconnect import DirectConnectFacade +from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade from ScoutSuite.providers.utils import run_concurrently +from ScoutSuite.core.console import print_error, print_debug +class AWSFacade(AWSBaseFacade): + def __init__(self, credentials: dict=None): + self._set_session(credentials) - -class AWSFacade(object): - - def __init__(self): - self.ec2 = EC2Facade() - self.awslambda = LambdaFacade() - self.cloudformation = CloudFormation() - self.cloudtrail = CloudTrailFacade() - self.cloudwatch = CloudWatch() - self.directconnect = DirectConnectFacade() - self.efs = EFSFacade() + self.ec2 = EC2Facade(self.session) + self.awslambda = LambdaFacade(self.session) + self.cloudformation = CloudFormation(self.session) + self.cloudtrail = CloudTrailFacade(self.session) + self.cloudwatch = CloudWatch(self.session) + self.directconnect = DirectConnectFacade(self.session) + self.efs = EFSFacade(self.session) async def build_region_list(self, service: str, chosen_regions=None, partition_name='aws'): service = 'ec2containerservice' if service == 'ecs' else service @@ -36,3 +38,25 @@ async def build_region_list(self, service: str, chosen_regions=None, partition_n return list((Counter(regions) & Counter(chosen_regions)).elements()) else: return regions + + def _set_session(self, credentials: dict): + # TODO: This conditional check is ok for now, but eventually, the credentials should always be provided. + if not credentials: + self.session = None + return + + session_params = {'aws_access_key_id': credentials.get('access_key'), + 'aws_secret_access_key': credentials.get('secret_key'), + 'aws_session_token': credentials.get('token')} + + self.session = boto3.session.Session(**session_params) + + # TODO: This should only be done in the constructor. I put this here for now, because this method is currently + # called from outside, but it should not happen. + self.ec2 = EC2Facade(self.session) + self.awslambda = LambdaFacade(self.session) + self.cloudformation = CloudFormation(self.session) + self.cloudtrail = CloudTrailFacade(self.session) + self.cloudwatch = CloudWatch(self.session) + self.directconnect = DirectConnectFacade(self.session) + self.efs = EFSFacade(self.session) diff --git a/ScoutSuite/providers/aws/facade/utils.py b/ScoutSuite/providers/aws/facade/utils.py index 6034494cb..94c0e189b 100644 --- a/ScoutSuite/providers/aws/facade/utils.py +++ b/ScoutSuite/providers/aws/facade/utils.py @@ -8,8 +8,8 @@ class AWSFacadeUtils: _clients = {} @staticmethod - async def get_all_pages(service: str, region: str, paginator_name: str, response_key: str, **paginator_args): - client = AWSFacadeUtils.get_client(service, region) + async def get_all_pages(service: str, region: str, session: boto3.session.Session, paginator_name: str, response_key: str, **paginator_args): + client = AWSFacadeUtils.get_client(service, region, session) # Building a paginator doesn't require any API call so no need to do it concurrently: paginator = client.get_paginator(paginator_name).paginate(**paginator_args) @@ -26,7 +26,17 @@ def _get_all_pages_from_paginator(paginator, key): return resources @staticmethod - def get_client(service: str, region: str): + def get_client(service: str, region: str, session: boto3.session.Session): + """ + Instantiates an AWS API client + + :param service: Service targeted, e.g. ec2 + :param session: The aws session + :param region: Region desired, e.g. us-east-2 + + :return: + """ + # TODO: investigate the use of a mutex to avoid useless creation of a same type of client among threads: - client = boto3.client(service, region_name=region) + client = session.client(service, region_name=region) return AWSFacadeUtils._clients.setdefault((service, region), client) diff --git a/ScoutSuite/providers/aws/resources/regions.py b/ScoutSuite/providers/aws/resources/regions.py index faf59865f..0d7c24ba4 100644 --- a/ScoutSuite/providers/aws/resources/regions.py +++ b/ScoutSuite/providers/aws/resources/regions.py @@ -13,6 +13,9 @@ def __init__(self, service): self.facade = AWSFacade() async def fetch_all(self, credentials, regions=None, partition_name='aws'): + # TODO: This should not be set here, the facade should be injected and already authenticated + self.facade._set_session(credentials) + self['regions'] = {} account_id = get_aws_account_id(credentials) for region in await self.facade.build_region_list(self.service, regions, partition_name): diff --git a/ScoutSuite/providers/aws/resources/resources.py b/ScoutSuite/providers/aws/resources/resources.py index 95361a738..ca87905ec 100644 --- a/ScoutSuite/providers/aws/resources/resources.py +++ b/ScoutSuite/providers/aws/resources/resources.py @@ -11,7 +11,7 @@ class AWSResources(Resources, metaclass=abc.ABCMeta): """This is the base class for AWS resources.""" - def __init__(self, scope: dict): + def __init__(self, facade, scope: dict): """ :param scope: The scope holds the scope in which the resource is located. This usually means \ at least a region, but can also contain a VPC id, an owner id, etc. It should be \ @@ -19,7 +19,7 @@ def __init__(self, scope: dict): """ self.scope = scope - self.facade = AWSFacade() + self.facade = facade class AWSCompositeResources(AWSResources, CompositeResources, metaclass=abc.ABCMeta): @@ -39,7 +39,7 @@ async def _fetch_children(self, parent: object, scope: dict): :param scope: The scope passed to the children constructors """ - children = [(child_class(scope), child_name) for (child_class, child_name) in self._children] + children = [(child_class(self.facade, scope), child_name) for (child_class, child_name) in self._children] # fetch all children concurrently: await asyncio.wait({asyncio.ensure_future(child.fetch_all()) for (child, _) in children}) # update parent content: From 53b9ae9818aed32f83822381f706448a8e68fb96 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 11 Mar 2019 10:23:45 -0400 Subject: [PATCH 315/667] Updated test --- tests/test-aws_resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test-aws_resources.py b/tests/test-aws_resources.py index 48d5ec194..ccb99852f 100644 --- a/tests/test-aws_resources.py +++ b/tests/test-aws_resources.py @@ -33,7 +33,7 @@ class TestAWSResources(TestCase): def test_aws_composite_resource(self): loop = asyncio.new_event_loop() - composite = DummyComposite({'region': 'some_region'}) + composite = DummyComposite(None, {'region': 'some_region'}) loop.run_until_complete(composite.fetch_all()) with open(os.path.join(self.test_dir, 'data/aws-resources/dummy_resources.json')) as f: From 6e32b21d14d86ae70a5e5323cf5f426c59c7525e Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 11 Mar 2019 11:54:11 -0400 Subject: [PATCH 316/667] Define a facade for the Azure Network service. --- ScoutSuite/providers/azure/facade/network.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 ScoutSuite/providers/azure/facade/network.py diff --git a/ScoutSuite/providers/azure/facade/network.py b/ScoutSuite/providers/azure/facade/network.py new file mode 100644 index 000000000..1b7ede9f8 --- /dev/null +++ b/ScoutSuite/providers/azure/facade/network.py @@ -0,0 +1,13 @@ +from azure.mgmt.network import NetworkManagementClient +from ScoutSuite.providers.utils import run_concurrently + + +class NetworkFacade: + def __init__(self, credentials, subscription_id): + self._client = NetworkManagementClient(credentials, subscription_id) + + async def get_network_watchers(self): + return await run_concurrently(self._client.network_watchers.list_all) + + async def get_network_security_groups(self): + return await run_concurrently(self._client.network_security_groups.list_all) From c6c49042e60d5ba138c8ae30de0b1c4b9c370ddd Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 11 Mar 2019 11:54:37 -0400 Subject: [PATCH 317/667] Define NetworkSecurityGroups resources. --- .../network/network_security_groups.py | 129 ++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 ScoutSuite/providers/azure/resources/network/network_security_groups.py diff --git a/ScoutSuite/providers/azure/resources/network/network_security_groups.py b/ScoutSuite/providers/azure/resources/network/network_security_groups.py new file mode 100644 index 000000000..bf7117450 --- /dev/null +++ b/ScoutSuite/providers/azure/resources/network/network_security_groups.py @@ -0,0 +1,129 @@ +from ScoutSuite.providers.base.configs.resources import Resources + + +class NetworkSecurityGroups(Resources): + + def __init__(self, facade): + self.facade = facade + + async def fetch_all(self): + for raw_group in await self.facade.get_network_security_groups(): + id, network_security_group = self._parse(raw_group) + self[id] = network_security_group + + def _parse(self, network_security_group): + network_security_group_dict = {} + network_security_group_dict['id'] = network_security_group.id + network_security_group_dict['name'] = network_security_group.name + network_security_group_dict['provisioning_state'] = network_security_group.provisioning_state + network_security_group_dict['location'] = network_security_group.location + network_security_group_dict['resource_guid'] = network_security_group.resource_guid + network_security_group_dict['etag'] = network_security_group.etag + + network_security_group_dict['security_rules'] = self._parse_security_rules(network_security_group) + + exposed_ports = self._parse_exposed_ports(network_security_group) + network_security_group_dict['exposed_ports'] = exposed_ports + network_security_group_dict['exposed_port_ranges'] = self._format_ports(exposed_ports) + + return network_security_group_dict['id'], network_security_group_dict + + def _parse_security_rules(self, network_security_group): + security_rules = {} + for sr in network_security_group.security_rules: + security_rule_dict = {} + security_rule_dict['id'] = sr.id + security_rule_dict['name'] = sr.name + security_rule_dict['allow'] = sr.access == "Allow" + security_rule_dict['priority'] = sr.priority + security_rule_dict['description'] = sr.description + security_rule_dict['provisioning_state'] = sr.provisioning_state + + security_rule_dict['protocol'] = sr.protocol + security_rule_dict['direction'] = sr.direction + + source_address_prefixes = self._merge_prefixes_or_ports(sr.source_address_prefix, + sr.source_address_prefixes) + security_rule_dict['source_address_prefixes'] = source_address_prefixes + + source_port_ranges = self._merge_prefixes_or_ports(sr.source_port_range, sr.source_port_ranges) + security_rule_dict['source_port_ranges'] = source_port_ranges + security_rule_dict['source_ports'] = self._parse_ports(source_port_ranges) + + destination_address_prefixes = self._merge_prefixes_or_ports(sr.destination_address_prefix, + sr.destination_address_prefixes) + security_rule_dict['destination_address_prefixes'] = destination_address_prefixes + + destination_port_ranges = self._merge_prefixes_or_ports(sr.destination_port_range, + sr.destination_port_ranges) + security_rule_dict['destination_port_ranges'] = destination_port_ranges + security_rule_dict['destination_ports'] = self._parse_ports(destination_port_ranges) + + security_rule_dict['etag'] = sr.etag + + security_rules[security_rule_dict['id']] = security_rule_dict + + return security_rules + + def _parse_ports(self, port_ranges): + ports = set() + for pr in port_ranges: + if pr == "*": + for p in range(0, 65535 + 1): + ports.add(p) + break + elif "-" in pr: + lower, upper = pr.split("-") + for p in range(int(lower), int(upper) + 1): + ports.add(p) + else: + ports.add(int(pr)) + ports = list(ports) + ports.sort() + return ports + + def _parse_exposed_ports(self, network_security_group): + exposed_ports = set() + + # Sort by priority. + rules = network_security_group.default_security_rules + network_security_group.security_rules + rules.sort(key=lambda x: x.priority, reverse=True) + + for sr in rules: + if sr.direction == "Inbound" and (sr.source_address_prefix == "*" + or sr.source_address_prefix == "Internet"): + port_ranges = self._merge_prefixes_or_ports(sr.destination_port_range, + sr.destination_port_ranges) + ports = self._parse_ports(port_ranges) + if sr.access == "Allow": + for p in ports: + exposed_ports.add(p) + else: + for p in ports: + exposed_ports.discard(p) + exposed_ports = list(exposed_ports) + exposed_ports.sort() + return exposed_ports + + def _merge_prefixes_or_ports(self, port_range, port_ranges): + port_ranges = port_ranges if port_ranges else [] + if port_range: + port_ranges.append(port_range) + return port_ranges + + def _format_ports(self, ports): + port_ranges = [] + start = None + for i in range(0, 65535 + 1): + if i in ports: + if not start: + start = i + else: + if start: + if i - 1 == start: + port_ranges.append(str(start)) + else: + port_ranges.append(str(start) + "-" + str(i - 1)) + start = None + return port_ranges + From f443662cad196feb2abfe29978a4ce9d72932484 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 11 Mar 2019 11:54:53 -0400 Subject: [PATCH 318/667] Define NetworkWatchers resources. --- .../resources/network/network_watchers.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 ScoutSuite/providers/azure/resources/network/network_watchers.py diff --git a/ScoutSuite/providers/azure/resources/network/network_watchers.py b/ScoutSuite/providers/azure/resources/network/network_watchers.py new file mode 100644 index 000000000..7548cf667 --- /dev/null +++ b/ScoutSuite/providers/azure/resources/network/network_watchers.py @@ -0,0 +1,22 @@ +from ScoutSuite.providers.base.configs.resources import Resources + + +class NetworkWatchers(Resources): + + def __init__(self, facade): + self.facade = facade + + async def fetch_all(self): + for raw_watcher in await self.facade.get_network_watchers(): + id, network_watcher = self._parse(raw_watcher) + self[id] = network_watcher + + def _parse(self, network_watcher): + network_watcher_dict = {} + network_watcher_dict['id'] = network_watcher.id + network_watcher_dict['name'] = network_watcher.name + network_watcher_dict['provisioning_state'] = network_watcher.provisioning_state + network_watcher_dict['location'] = network_watcher.location + network_watcher_dict['etag'] = network_watcher.etag + + return network_watcher_dict['id'], network_watcher_dict From cbd6f2739a95902d524dad3196c730ea98c5b950 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 11 Mar 2019 11:55:20 -0400 Subject: [PATCH 319/667] Define Networks resources. --- .../azure/resources/network/networks.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 ScoutSuite/providers/azure/resources/network/networks.py diff --git a/ScoutSuite/providers/azure/resources/network/networks.py b/ScoutSuite/providers/azure/resources/network/networks.py new file mode 100644 index 000000000..e872d5b44 --- /dev/null +++ b/ScoutSuite/providers/azure/resources/network/networks.py @@ -0,0 +1,21 @@ +from ScoutSuite.providers.azure.resources.resources import AzureCompositeResources +from ScoutSuite.providers.azure.facade.network import NetworkFacade + +from .network_security_groups import NetworkSecurityGroups +from .network_watchers import NetworkWatchers + + +class Networks(AzureCompositeResources): + _children = [ + (NetworkSecurityGroups, 'network_security_groups'), + (NetworkWatchers, 'network_watchers') + ] + + async def fetch_all(self, credentials, **kwargs): + # TODO: build that facade somewhere else: + facade = NetworkFacade(credentials.credentials, credentials.subscription_id) + + await self._fetch_children(parent=self, facade=facade) + + self['network_security_groups_count'] = len(self['network_security_groups']) + self['network_watchers_count'] = len(self['network_watchers']) From 2eab127d7af0507562e0650fb9f3d83f0d313392 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Mon, 11 Mar 2019 12:29:22 -0400 Subject: [PATCH 320/667] Fixed dict stuff --- ScoutSuite/providers/aws/provider.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index 5ea7ad86f..3bb610562 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -111,13 +111,11 @@ def _add_security_group_data_to_elbv2(self): for region in elbv2_config['regions']: for vpc in elbv2_config['regions'][region]['vpcs']: for lb in elbv2_config['regions'][region]['vpcs'][vpc]['lbs']: - security_groups = ec2_config['regions'][region]['vpcs'][vpc]['security_groups'] for i in range(0, len(elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'])): - for security_group in security_groups: - if elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][i]['GroupId'] \ - is security_group['id']: + for sg in ec2_config['regions'][region]['vpcs'][vpc]['security_groups']: + if elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][i]['GroupId'] == sg: elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][i] = \ - security_group + ec2_config['regions'][region]['vpcs'][vpc]['security_groups'][sg] def _check_ec2_zone_distribution(self): regions = self.services['ec2']['regions'].values() From debf9872b0d37e11f1f54aa9cdf79c1825c80da7 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 11 Mar 2019 12:32:21 -0400 Subject: [PATCH 321/667] Define a facade for the Azure Security Center service. --- .../providers/azure/facade/securitycenter.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 ScoutSuite/providers/azure/facade/securitycenter.py diff --git a/ScoutSuite/providers/azure/facade/securitycenter.py b/ScoutSuite/providers/azure/facade/securitycenter.py new file mode 100644 index 000000000..a1625f7ed --- /dev/null +++ b/ScoutSuite/providers/azure/facade/securitycenter.py @@ -0,0 +1,16 @@ +from azure.mgmt.security import SecurityCenter +from ScoutSuite.providers.utils import run_concurrently + + +class SecurityCenterFacade: + def __init__(self, credentials, subscription_id): + self._client = SecurityCenter(credentials, subscription_id, '') + + async def get_pricings(self): + return await run_concurrently(self._client.pricings.list) + + async def get_security_contacts(self): + return await run_concurrently(self._client.security_contacts.list) + + async def get_auto_provisioning_settings(self): + return await run_concurrently(self._client.auto_provisioning_settings.list) From 3fc3222622f54fb68bf5c16f7053426e75ad32a6 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 11 Mar 2019 12:32:41 -0400 Subject: [PATCH 322/667] Define AutoProvisioningSettings resources. --- .../auto_provisioning_settings.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 ScoutSuite/providers/azure/resources/securitycenter/auto_provisioning_settings.py diff --git a/ScoutSuite/providers/azure/resources/securitycenter/auto_provisioning_settings.py b/ScoutSuite/providers/azure/resources/securitycenter/auto_provisioning_settings.py new file mode 100644 index 000000000..19a5919b4 --- /dev/null +++ b/ScoutSuite/providers/azure/resources/securitycenter/auto_provisioning_settings.py @@ -0,0 +1,20 @@ +from ScoutSuite.providers.base.configs.resources import Resources + + +class AutoProvisioningSettings(Resources): + + def __init__(self, facade): + self.facade = facade + + async def fetch_all(self): + for raw_settings in await self.facade.get_auto_provisioning_settings(): + id, auto_provisioning_settings = self._parse(raw_settings) + self[id] = auto_provisioning_settings + + def _parse(self, auto_provisioning_setting): + auto_provisioning_setting_dict = {} + auto_provisioning_setting_dict['id'] = auto_provisioning_setting.id + auto_provisioning_setting_dict['name'] = auto_provisioning_setting.name + auto_provisioning_setting_dict['auto_provision'] = auto_provisioning_setting.auto_provision + + return auto_provisioning_setting_dict['id'], auto_provisioning_setting_dict From ebb00dd8e7906aaf295cc506c031371ae6e46e43 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 11 Mar 2019 12:32:54 -0400 Subject: [PATCH 323/667] Define Pricings resources. --- .../resources/securitycenter/pricings.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 ScoutSuite/providers/azure/resources/securitycenter/pricings.py diff --git a/ScoutSuite/providers/azure/resources/securitycenter/pricings.py b/ScoutSuite/providers/azure/resources/securitycenter/pricings.py new file mode 100644 index 000000000..d87bfe2b8 --- /dev/null +++ b/ScoutSuite/providers/azure/resources/securitycenter/pricings.py @@ -0,0 +1,20 @@ +from ScoutSuite.providers.base.configs.resources import Resources + + +class Pricings(Resources): + + def __init__(self, facade): + self.facade = facade + + async def fetch_all(self): + for raw_pricing in await self.facade.get_pricings(): + id, pricing = self._parse(raw_pricing) + self[id] = pricing + + def _parse(self, pricing): + pricing_dict = {} + pricing_dict['id'] = pricing.id + pricing_dict['name'] = pricing.name + pricing_dict['pricing_tier'] = pricing.pricing_tier + + return pricing_dict['id'], pricing_dict From 6b3ceb80f7eca636ca0629a42ef208c4d9a8392f Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 11 Mar 2019 12:33:10 -0400 Subject: [PATCH 324/667] Define SecurityContacts resources. --- .../securitycenter/security_contacts.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 ScoutSuite/providers/azure/resources/securitycenter/security_contacts.py diff --git a/ScoutSuite/providers/azure/resources/securitycenter/security_contacts.py b/ScoutSuite/providers/azure/resources/securitycenter/security_contacts.py new file mode 100644 index 000000000..a7f9b23ea --- /dev/null +++ b/ScoutSuite/providers/azure/resources/securitycenter/security_contacts.py @@ -0,0 +1,24 @@ +from ScoutSuite.providers.base.configs.resources import Resources + + +class SecurityContacts(Resources): + + def __init__(self, facade): + self.facade = facade + + async def fetch_all(self): + for raw_contact in await self.facade.get_security_contacts(): + id, security_contact = self._parse(raw_contact) + self[id] = security_contact + + def _parse(self, security_contact): + security_contact_dict = {} + security_contact_dict['id'] = security_contact.id + security_contact_dict['name'] = security_contact.name + security_contact_dict['email'] = security_contact.email + security_contact_dict['phone'] = security_contact.phone + security_contact_dict['alert_notifications'] = security_contact.alert_notifications == "On" + security_contact_dict['alerts_to_admins'] = security_contact.alerts_to_admins == "On" + security_contact_dict['additional_properties'] = security_contact.additional_properties + + return security_contact_dict['id'], security_contact_dict From 02eb22bd236e0f5f7789fccd12ffe1132a701ec7 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 11 Mar 2019 12:33:30 -0400 Subject: [PATCH 325/667] Define SecurityCenter composite resource. --- .../securitycenter/security_center.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 ScoutSuite/providers/azure/resources/securitycenter/security_center.py diff --git a/ScoutSuite/providers/azure/resources/securitycenter/security_center.py b/ScoutSuite/providers/azure/resources/securitycenter/security_center.py new file mode 100644 index 000000000..6b9109f2e --- /dev/null +++ b/ScoutSuite/providers/azure/resources/securitycenter/security_center.py @@ -0,0 +1,24 @@ +from ScoutSuite.providers.azure.resources.resources import AzureCompositeResources +from ScoutSuite.providers.azure.facade.securitycenter import SecurityCenterFacade + +from .auto_provisioning_settings import AutoProvisioningSettings +from .pricings import Pricings +from .security_contacts import SecurityContacts + + +class SecurityCenter(AzureCompositeResources): + _children = [ + (AutoProvisioningSettings, 'auto_provisioning_settings'), + (Pricings, 'pricings'), + (SecurityContacts, 'security_contacts') + ] + + async def fetch_all(self, credentials, **kwargs): + # TODO: build that facade somewhere else: + facade = SecurityCenterFacade(credentials.credentials, credentials.subscription_id) + + await self._fetch_children(parent=self, facade=facade) + + self['auto_provisioning_settings_count'] = len(self['auto_provisioning_settings']) + self['pricings_count'] = len(self['pricings']) + self['security_contacts_count'] = len(self['security_contacts']) From fa10d0f8108bed0589271aedf7ad3d5efce446d2 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 11 Mar 2019 14:25:34 -0400 Subject: [PATCH 326/667] Migrate Azure Key Vault service to the new architecture. --- ScoutSuite/providers/azure/configs/services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/azure/configs/services.py b/ScoutSuite/providers/azure/configs/services.py index 9bf5da7fc..2de8097c5 100644 --- a/ScoutSuite/providers/azure/configs/services.py +++ b/ScoutSuite/providers/azure/configs/services.py @@ -6,7 +6,7 @@ from ScoutSuite.providers.azure.resources.sqldatabase.servers import Servers as SQLDatabaseConfig from ScoutSuite.providers.azure.services.securitycenter import SecurityCenterConfig from ScoutSuite.providers.azure.services.network import NetworkConfig -from ScoutSuite.providers.azure.services.keyvault import KeyVaultConfig +from ScoutSuite.providers.azure.resources.keyvault.key_vaults import KeyVaults as KeyVaultConfig try: from ScoutSuite.providers.azure.services.appgateway_private import AppGatewayConfig except ImportError: @@ -34,7 +34,7 @@ def __init__(self, metadata=None, thread_config=4, **kwargs): self.sqldatabase = SQLDatabaseConfig() self.securitycenter = SecurityCenterConfig(thread_config=thread_config) self.network = NetworkConfig(thread_config=thread_config) - self.keyvault = KeyVaultConfig(thread_config=thread_config) + self.keyvault = KeyVaultConfig() try: self.appgateway = AppGatewayConfig(thread_config=thread_config) From 16772a4e1b3d8231dd29a84160f3e75fc2409d54 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 11 Mar 2019 14:30:23 -0400 Subject: [PATCH 327/667] Migrate Azure Monitor service to the new architecture. --- ScoutSuite/providers/azure/configs/services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/azure/configs/services.py b/ScoutSuite/providers/azure/configs/services.py index 9bf5da7fc..a307cd4a5 100644 --- a/ScoutSuite/providers/azure/configs/services.py +++ b/ScoutSuite/providers/azure/configs/services.py @@ -2,7 +2,7 @@ from ScoutSuite.providers.base.configs.services import BaseServicesConfig from ScoutSuite.providers.azure.services.storageaccounts import StorageAccountsConfig -from ScoutSuite.providers.azure.services.monitor import MonitorConfig +from ScoutSuite.providers.azure.resources.monitor.activity_logs import ActivityLogs as MonitorConfig from ScoutSuite.providers.azure.resources.sqldatabase.servers import Servers as SQLDatabaseConfig from ScoutSuite.providers.azure.services.securitycenter import SecurityCenterConfig from ScoutSuite.providers.azure.services.network import NetworkConfig @@ -30,7 +30,7 @@ class AzureServicesConfig(BaseServicesConfig): def __init__(self, metadata=None, thread_config=4, **kwargs): self.storageaccounts = StorageAccountsConfig(thread_config=thread_config) - self.monitor = MonitorConfig(thread_config=thread_config) + self.monitor = MonitorConfig() self.sqldatabase = SQLDatabaseConfig() self.securitycenter = SecurityCenterConfig(thread_config=thread_config) self.network = NetworkConfig(thread_config=thread_config) From a2a7d276cac527a2042b3962d6601445cf715c16 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 11 Mar 2019 14:31:42 -0400 Subject: [PATCH 328/667] Migrate Azure network service to the new architecture. --- ScoutSuite/providers/azure/configs/services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/azure/configs/services.py b/ScoutSuite/providers/azure/configs/services.py index 9bf5da7fc..2be288603 100644 --- a/ScoutSuite/providers/azure/configs/services.py +++ b/ScoutSuite/providers/azure/configs/services.py @@ -5,7 +5,7 @@ from ScoutSuite.providers.azure.services.monitor import MonitorConfig from ScoutSuite.providers.azure.resources.sqldatabase.servers import Servers as SQLDatabaseConfig from ScoutSuite.providers.azure.services.securitycenter import SecurityCenterConfig -from ScoutSuite.providers.azure.services.network import NetworkConfig +from ScoutSuite.providers.azure.resources.network.networks import Networks as NetworkConfig from ScoutSuite.providers.azure.services.keyvault import KeyVaultConfig try: from ScoutSuite.providers.azure.services.appgateway_private import AppGatewayConfig @@ -33,7 +33,7 @@ def __init__(self, metadata=None, thread_config=4, **kwargs): self.monitor = MonitorConfig(thread_config=thread_config) self.sqldatabase = SQLDatabaseConfig() self.securitycenter = SecurityCenterConfig(thread_config=thread_config) - self.network = NetworkConfig(thread_config=thread_config) + self.network = NetworkConfig() self.keyvault = KeyVaultConfig(thread_config=thread_config) try: From 2dca539e677767aa02a16cb06945cfc6d15a6e0b Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 11 Mar 2019 14:33:14 -0400 Subject: [PATCH 329/667] Migrate Azure Security Center service to the new architecture. --- ScoutSuite/providers/azure/configs/services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/azure/configs/services.py b/ScoutSuite/providers/azure/configs/services.py index 9bf5da7fc..7876e23c5 100644 --- a/ScoutSuite/providers/azure/configs/services.py +++ b/ScoutSuite/providers/azure/configs/services.py @@ -4,7 +4,7 @@ from ScoutSuite.providers.azure.services.storageaccounts import StorageAccountsConfig from ScoutSuite.providers.azure.services.monitor import MonitorConfig from ScoutSuite.providers.azure.resources.sqldatabase.servers import Servers as SQLDatabaseConfig -from ScoutSuite.providers.azure.services.securitycenter import SecurityCenterConfig +from ScoutSuite.providers.azure.resources.securitycenter.security_center import SecurityCenter as SecurityCenterConfig from ScoutSuite.providers.azure.services.network import NetworkConfig from ScoutSuite.providers.azure.services.keyvault import KeyVaultConfig try: @@ -32,7 +32,7 @@ def __init__(self, metadata=None, thread_config=4, **kwargs): self.storageaccounts = StorageAccountsConfig(thread_config=thread_config) self.monitor = MonitorConfig(thread_config=thread_config) self.sqldatabase = SQLDatabaseConfig() - self.securitycenter = SecurityCenterConfig(thread_config=thread_config) + self.securitycenter = SecurityCenterConfig() self.network = NetworkConfig(thread_config=thread_config) self.keyvault = KeyVaultConfig(thread_config=thread_config) From f56851f25fc46ff9e8bb39cc3bac469c1829c738 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 11 Mar 2019 14:34:43 -0400 Subject: [PATCH 330/667] Add id_suffix and update HTML template. --- .../azure/services.storageaccounts.storage_accounts.html | 1 + .../findings/storageaccount-account-allowing-clear-text.json | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/output/data/html/partials/azure/services.storageaccounts.storage_accounts.html b/ScoutSuite/output/data/html/partials/azure/services.storageaccounts.storage_accounts.html index 2da2896af..00892a6f2 100644 --- a/ScoutSuite/output/data/html/partials/azure/services.storageaccounts.storage_accounts.html +++ b/ScoutSuite/output/data/html/partials/azure/services.storageaccounts.storage_accounts.html @@ -7,6 +7,7 @@

    {{name}}

    Information

    Storage Account Name: {{name}}
    +
    HTTPS required: {{https_traffic_enabled}}
    Public traffic: {{ convert_bool_to_enabled public_traffic_allowed }}
    Access Key Rotated: {{ get_value_at 'services' 'monitor' 'activity_logs' 'storage_accounts' @key 'access_keys_rotated' }}
    diff --git a/ScoutSuite/providers/azure/rules/findings/storageaccount-account-allowing-clear-text.json b/ScoutSuite/providers/azure/rules/findings/storageaccount-account-allowing-clear-text.json index 534c162c3..18b204af7 100644 --- a/ScoutSuite/providers/azure/rules/findings/storageaccount-account-allowing-clear-text.json +++ b/ScoutSuite/providers/azure/rules/findings/storageaccount-account-allowing-clear-text.json @@ -5,5 +5,6 @@ "dashboard_name": "Accounts", "conditions": [ "and", [ "storageaccounts.storage_accounts.id.https_traffic_enabled", "false", "" ] - ] + ], + "id_suffix": "https_traffic_enabled" } From 2f770405d1d13b9b062d9ce4944e31be31cd5478 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 11 Mar 2019 14:34:53 -0400 Subject: [PATCH 331/667] Migrate storageaccounts service to the new architecture. --- ScoutSuite/providers/azure/configs/services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/azure/configs/services.py b/ScoutSuite/providers/azure/configs/services.py index 9bf5da7fc..ef23fb512 100644 --- a/ScoutSuite/providers/azure/configs/services.py +++ b/ScoutSuite/providers/azure/configs/services.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- from ScoutSuite.providers.base.configs.services import BaseServicesConfig -from ScoutSuite.providers.azure.services.storageaccounts import StorageAccountsConfig from ScoutSuite.providers.azure.services.monitor import MonitorConfig from ScoutSuite.providers.azure.resources.sqldatabase.servers import Servers as SQLDatabaseConfig +from ScoutSuite.providers.azure.resources.storageaccounts.storageaccounts import StorageAccounts as StorageAccountsConfig from ScoutSuite.providers.azure.services.securitycenter import SecurityCenterConfig from ScoutSuite.providers.azure.services.network import NetworkConfig from ScoutSuite.providers.azure.services.keyvault import KeyVaultConfig @@ -29,7 +29,7 @@ class AzureServicesConfig(BaseServicesConfig): def __init__(self, metadata=None, thread_config=4, **kwargs): - self.storageaccounts = StorageAccountsConfig(thread_config=thread_config) + self.storageaccounts = StorageAccountsConfig() self.monitor = MonitorConfig(thread_config=thread_config) self.sqldatabase = SQLDatabaseConfig() self.securitycenter = SecurityCenterConfig(thread_config=thread_config) From ee1862593b54256d5ecc70a279e84c41f61ce004 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 11 Mar 2019 15:33:54 -0400 Subject: [PATCH 332/667] Migrate all Azure private services to the new architecture. --- ScoutSuite/providers/azure/configs/services.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ScoutSuite/providers/azure/configs/services.py b/ScoutSuite/providers/azure/configs/services.py index 9bf5da7fc..d5c537e37 100644 --- a/ScoutSuite/providers/azure/configs/services.py +++ b/ScoutSuite/providers/azure/configs/services.py @@ -8,19 +8,19 @@ from ScoutSuite.providers.azure.services.network import NetworkConfig from ScoutSuite.providers.azure.services.keyvault import KeyVaultConfig try: - from ScoutSuite.providers.azure.services.appgateway_private import AppGatewayConfig + from ScoutSuite.providers.azure.resources.appgateway_private.application_gateways_private import ApplicationGateways except ImportError: pass try: - from ScoutSuite.providers.azure.services.rediscache_private import RedisCacheConfig + from ScoutSuite.providers.azure.resources.rediscache_private.redis_caches_private import RedisCaches except ImportError: pass try: - from ScoutSuite.providers.azure.services.appservice_private import AppServiceConfig + from ScoutSuite.providers.azure.resources.appservice_private.web_applications_private import WebApplications except ImportError: pass try: - from ScoutSuite.providers.azure.services.loadbalancer_private import LoadBalancerConfig + from ScoutSuite.providers.azure.resources.loadbalancer_private.load_balancers_private import LoadBalancers except ImportError: pass @@ -37,19 +37,19 @@ def __init__(self, metadata=None, thread_config=4, **kwargs): self.keyvault = KeyVaultConfig(thread_config=thread_config) try: - self.appgateway = AppGatewayConfig(thread_config=thread_config) + self.appgateway = ApplicationGateways() except NameError: pass try: - self.rediscache = RedisCacheConfig(thread_config=thread_config) + self.rediscache = RedisCaches() except NameError: pass try: - self.appservice = AppServiceConfig(thread_config=thread_config) + self.appservice = WebApplications() except NameError: pass try: - self.loadbalancer = LoadBalancerConfig(thread_config=thread_config) + self.loadbalancer = LoadBalancers() except NameError: pass From 0ccb32df9e4041701c0d2e61dcd134fe3921a166 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 11 Mar 2019 17:04:39 -0400 Subject: [PATCH 333/667] Implemented ElastiCache interface --- .../providers/aws/facade/elasticache.py | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 ScoutSuite/providers/aws/facade/elasticache.py diff --git a/ScoutSuite/providers/aws/facade/elasticache.py b/ScoutSuite/providers/aws/facade/elasticache.py new file mode 100644 index 000000000..5e3c592c3 --- /dev/null +++ b/ScoutSuite/providers/aws/facade/elasticache.py @@ -0,0 +1,69 @@ +import boto3 + +from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils +from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade +from ScoutSuite.providers.aws.utils import ec2_classic +from asyncio import Lock + +class ElastiCacheFacade(AWSBaseFacade): + regional_clusters_cache_locks = {} + regional_subnets_cache_locks = {} + clusters_cache = {} + subnets_cache = {} + + async def get_clusters(self, region, vpc): + # This result could be cached to save some api calls + await self.cache_clusters(region) + + return [cluster for cluster in self.clusters_cache[region] if cluster['VpcId'] == vpc] + + async def cache_clusters(self, region): + async with self.regional_clusters_cache_locks.setdefault(region, Lock()): + if region in self.clusters_cache: + return + + self.clusters_cache[region] = await AWSFacadeUtils.get_all_pages('elasticache', region, self.session, 'describe_cache_clusters', 'CacheClusters') + + for cluster in self.clusters_cache[region]: + if 'CacheSubnetGroupName' not in cluster: + cluster['VpcId'] = ec2_classic + else: + subnet_group = await self.get_subnet_group(region, cluster['CacheSubnetGroupName']) + cluster['VpcId'] = subnet_group['VpcId'] + + async def get_security_groups(self, region): + client = AWSFacadeUtils.get_client('elasticache', region, self.session) + + try: + return await AWSFacadeUtils.get_all_pages('elasticache', region, self.session, 'describe_cache_security_groups', 'CacheSecurityGroups') + + except client.exceptions.InvalidParameterValueException: + # Recent account are not allowed to use security groups at this level. Calling + # describe_cache_security_groups will throw an InvalidParameterValueException exception. + pass + + return [] + + async def get_subnet_groups(self, region, vpc): + await self.cache_subnets(region) + return [subnet for subnet in self.subnets_cache[region] if subnet['VpcId'] == vpc] + + async def get_subnet_group(self, region, subnet_name): + subnets = await AWSFacadeUtils.get_all_pages('elasticache', \ + region, \ + self.session, \ + 'describe_cache_subnet_groups', \ + 'CacheSubnetGroups', \ + CacheSubnetGroupName=subnet_name \ + ) + return subnets[0] + + async def cache_subnets(self, region): + async with self.regional_subnets_cache_locks.setdefault(region, Lock()): + if region in self.subnets_cache: + return + + self.subnets_cache[region] = await AWSFacadeUtils.get_all_pages('elasticache', region, self.session, 'describe_cache_subnet_groups', 'CacheSubnetGroups') + + async def get_parameter_groups(self, region): + return await AWSFacadeUtils.get_all_pages('elasticache', region, self.session, 'describe_cache_parameter_groups', 'CacheParameterGroups') \ No newline at end of file From 79710d97bab6225866a3c689841535e4ef463126 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 11 Mar 2019 17:06:05 -0400 Subject: [PATCH 334/667] Sorted imports a bit --- ScoutSuite/providers/aws/facade/facade.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/facade/facade.py b/ScoutSuite/providers/aws/facade/facade.py index 430909313..dd0d2f13b 100644 --- a/ScoutSuite/providers/aws/facade/facade.py +++ b/ScoutSuite/providers/aws/facade/facade.py @@ -6,9 +6,10 @@ from ScoutSuite.providers.aws.facade.cloudformation import CloudFormation from ScoutSuite.providers.aws.facade.cloudtrail import CloudTrailFacade from ScoutSuite.providers.aws.facade.cloudwatch import CloudWatch +from ScoutSuite.providers.aws.facade.directconnect import DirectConnectFacade from ScoutSuite.providers.aws.facade.ec2 import EC2Facade from ScoutSuite.providers.aws.facade.efs import EFSFacade -from ScoutSuite.providers.aws.facade.directconnect import DirectConnectFacade +from ScoutSuite.providers.aws.facade.elasticache import ElastiCacheFacade from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade from ScoutSuite.providers.utils import run_concurrently from ScoutSuite.core.console import print_error, print_debug From 48f2e2ef7ef93655b8c63b8036223300c6bee473 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 11 Mar 2019 17:06:39 -0400 Subject: [PATCH 335/667] Lint --- ScoutSuite/providers/aws/facade/facade.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/facade.py b/ScoutSuite/providers/aws/facade/facade.py index dd0d2f13b..1e99d3cd6 100644 --- a/ScoutSuite/providers/aws/facade/facade.py +++ b/ScoutSuite/providers/aws/facade/facade.py @@ -14,8 +14,9 @@ from ScoutSuite.providers.utils import run_concurrently from ScoutSuite.core.console import print_error, print_debug + class AWSFacade(AWSBaseFacade): - def __init__(self, credentials: dict=None): + def __init__(self, credentials: dict = None): self._set_session(credentials) self.ec2 = EC2Facade(self.session) @@ -51,7 +52,7 @@ def _set_session(self, credentials: dict): 'aws_session_token': credentials.get('token')} self.session = boto3.session.Session(**session_params) - + # TODO: This should only be done in the constructor. I put this here for now, because this method is currently # called from outside, but it should not happen. self.ec2 = EC2Facade(self.session) From d47df313d03c5754f5976ded7791bcbcf918c977 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 11 Mar 2019 17:06:48 -0400 Subject: [PATCH 336/667] Integrated ElastiCache facade --- ScoutSuite/providers/aws/facade/facade.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ScoutSuite/providers/aws/facade/facade.py b/ScoutSuite/providers/aws/facade/facade.py index 1e99d3cd6..e0feccb46 100644 --- a/ScoutSuite/providers/aws/facade/facade.py +++ b/ScoutSuite/providers/aws/facade/facade.py @@ -26,6 +26,7 @@ def __init__(self, credentials: dict = None): self.cloudwatch = CloudWatch(self.session) self.directconnect = DirectConnectFacade(self.session) self.efs = EFSFacade(self.session) + self.elasticache = ElastiCacheFacade(self.session) async def build_region_list(self, service: str, chosen_regions=None, partition_name='aws'): service = 'ec2containerservice' if service == 'ecs' else service @@ -62,3 +63,4 @@ def _set_session(self, credentials: dict): self.cloudwatch = CloudWatch(self.session) self.directconnect = DirectConnectFacade(self.session) self.efs = EFSFacade(self.session) + self.elasticache = ElastiCacheFacade(self.session) From 361a8466cc90721d4c0a54764cd3c1954e6871ff Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 11 Mar 2019 17:07:17 -0400 Subject: [PATCH 337/667] Generalized the cluster class --- .../providers/aws/resources/ec2/vpcs.py | 29 ++------------- ScoutSuite/providers/aws/resources/vpcs.py | 35 +++++++++++++++++++ 2 files changed, 38 insertions(+), 26 deletions(-) create mode 100644 ScoutSuite/providers/aws/resources/vpcs.py diff --git a/ScoutSuite/providers/aws/resources/ec2/vpcs.py b/ScoutSuite/providers/aws/resources/ec2/vpcs.py index b8b520059..c3125df21 100644 --- a/ScoutSuite/providers/aws/resources/ec2/vpcs.py +++ b/ScoutSuite/providers/aws/resources/ec2/vpcs.py @@ -1,37 +1,14 @@ import asyncio -from ScoutSuite.providers.aws.resources.resources import AWSCompositeResources +from ScoutSuite.providers.aws.resources.vpcs import Vpcs from ScoutSuite.providers.aws.resources.ec2.instances import EC2Instances from ScoutSuite.providers.aws.resources.ec2.securitygroups import SecurityGroups from ScoutSuite.providers.aws.resources.ec2.networkinterfaces import NetworkInterfaces -class Vpcs(AWSCompositeResources): +class Ec2Vpcs(Vpcs): _children = [ (EC2Instances, 'instances'), (SecurityGroups, 'security_groups'), (NetworkInterfaces, 'network_interfaces') - ] - - async def fetch_all(self, **kwargs): - vpcs = await self.facade.ec2.get_vpcs(self.scope['region']) - for vpc in vpcs: - name, resource = self._parse_vpc(vpc) - self[name] = resource - - # TODO: make a refactoring of the following: - if len(self) == 0: - return - tasks = { - asyncio.ensure_future( - self._fetch_children( - self[vpc], - {'region': self.scope['region'], 'vpc': vpc} - ) - ) for vpc in self - } - await asyncio.wait(tasks) - - def _parse_vpc(self, vpc): - return vpc['VpcId'], {} - + ] \ No newline at end of file diff --git a/ScoutSuite/providers/aws/resources/vpcs.py b/ScoutSuite/providers/aws/resources/vpcs.py new file mode 100644 index 000000000..d4bf9d879 --- /dev/null +++ b/ScoutSuite/providers/aws/resources/vpcs.py @@ -0,0 +1,35 @@ +import asyncio + +from ScoutSuite.providers.aws.resources.resources import AWSCompositeResources + + +class Vpcs(AWSCompositeResources): + """ + Fetches resources inside the virtual private clouds (VPCs) defined in a region. + :param add_ec2_classic: Setting this parameter to True will add 'EC2-Classic' to the list of VPCs. + """ + + def __init__(self, facade, scope: dict, add_ec2_classic=False): + super(Vpcs, self).__init__(facade, scope) + self.add_ec2_classic = add_ec2_classic + + async def fetch_all(self, **kwargs): + vpcs = await self.facade.ec2.get_vpcs(self.scope['region']) + for vpc in vpcs: + name, resource = self._parse_vpc(vpc) + self[name] = resource + + # TODO: make a refactoring of the following: + if len(self) == 0: + return + + tasks = { + asyncio.ensure_future( + self._fetch_children(self[vpc], {'region': self.scope['region'], 'vpc': vpc}) + ) for vpc in self + } + + await asyncio.wait(tasks) + + def _parse_vpc(self, vpc): + return vpc['VpcId'], {} From 63faef5cd181c4d8ad3d04d49b0b9ceb7d47c965 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 11 Mar 2019 17:21:27 -0400 Subject: [PATCH 338/667] Implemented ElastiCache subnet groups --- .../aws/resources/elasticache/subnetgroups.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 ScoutSuite/providers/aws/resources/elasticache/subnetgroups.py diff --git a/ScoutSuite/providers/aws/resources/elasticache/subnetgroups.py b/ScoutSuite/providers/aws/resources/elasticache/subnetgroups.py new file mode 100644 index 000000000..8ef864cb7 --- /dev/null +++ b/ScoutSuite/providers/aws/resources/elasticache/subnetgroups.py @@ -0,0 +1,14 @@ +from ScoutSuite.providers.aws.resources.resources import AWSCompositeResources +from ScoutSuite.providers.aws.utils import ec2_classic + + +class SubnetGroups(AWSCompositeResources): + async def fetch_all(self, **kwargs): + raw_subnet_groups = await self.facade.elasticache.get_subnet_groups(self.scope['region'], self.scope['vpc']) + for raw_subnet_group in raw_subnet_groups: + name, resource = self._parse_subnet_group(raw_subnet_group) + self[name] = resource + + def _parse_subnet_group(self, raw_subnet_group): + raw_subnet_group['name'] = raw_subnet_group.pop('CacheSubnetGroupName') + return raw_subnet_group['name'], raw_subnet_group \ No newline at end of file From 78a5baa556d57501a4d8d85abcf2880ad6fba4a9 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 11 Mar 2019 17:21:39 -0400 Subject: [PATCH 339/667] Implemented ElastiCache clusters --- .../providers/aws/resources/elasticache/cluster.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 ScoutSuite/providers/aws/resources/elasticache/cluster.py diff --git a/ScoutSuite/providers/aws/resources/elasticache/cluster.py b/ScoutSuite/providers/aws/resources/elasticache/cluster.py new file mode 100644 index 000000000..2ac367378 --- /dev/null +++ b/ScoutSuite/providers/aws/resources/elasticache/cluster.py @@ -0,0 +1,14 @@ +from ScoutSuite.providers.aws.resources.resources import AWSCompositeResources +from ScoutSuite.providers.aws.utils import ec2_classic + + +class Clusters(AWSCompositeResources): + async def fetch_all(self, **kwargs): + raw_clusters = await self.facade.elasticache.get_clusters(self.scope['region'], self.scope['vpc']) + for raw_cluster in raw_clusters: + name, resource = self._parse_cluster(raw_cluster) + self[name] = resource + + def _parse_cluster(self, raw_cluster): + raw_cluster['name'] = raw_cluster.pop('CacheClusterId') + return raw_cluster['name'], raw_cluster \ No newline at end of file From 8d4e069690f61316024793945d9da37ad6294010 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 11 Mar 2019 17:21:53 -0400 Subject: [PATCH 340/667] Implemented ElastiCache Vpcs --- .../providers/aws/resources/elasticache/vpcs.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 ScoutSuite/providers/aws/resources/elasticache/vpcs.py diff --git a/ScoutSuite/providers/aws/resources/elasticache/vpcs.py b/ScoutSuite/providers/aws/resources/elasticache/vpcs.py new file mode 100644 index 000000000..666c6bb7b --- /dev/null +++ b/ScoutSuite/providers/aws/resources/elasticache/vpcs.py @@ -0,0 +1,16 @@ +import asyncio + +from ScoutSuite.providers.aws.resources.vpcs import Vpcs +from ScoutSuite.providers.aws.resources.elasticache.cluster import Clusters +from ScoutSuite.providers.aws.resources.elasticache.subnetgroups import SubnetGroups +from ScoutSuite.providers.aws.utils import ec2_classic + + +class ElastiCacheVpcs(Vpcs): + _children = [ + (Clusters, 'clusters'), + (SubnetGroups, 'subnet_groups') + ] + + def __init__(self, facade, scope: dict): + super(ElastiCacheVpcs, self).__init__(facade, scope, add_ec2_classic=True) \ No newline at end of file From efb05c0d8b434634512f274236570201c2269fce Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 11 Mar 2019 17:29:36 -0400 Subject: [PATCH 341/667] Implemented ElastiCache parameter groups --- .../aws/resources/elasticache/parametergroups.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 ScoutSuite/providers/aws/resources/elasticache/parametergroups.py diff --git a/ScoutSuite/providers/aws/resources/elasticache/parametergroups.py b/ScoutSuite/providers/aws/resources/elasticache/parametergroups.py new file mode 100644 index 000000000..fb7223d17 --- /dev/null +++ b/ScoutSuite/providers/aws/resources/elasticache/parametergroups.py @@ -0,0 +1,13 @@ +from ScoutSuite.providers.aws.resources.resources import AWSResources + + +class ParameterGroups(AWSResources): + async def fetch_all(self, **kwargs): + raw_parameter_groups = await self.facade.elasticache.get_parameter_groups(self.scope['region']) + for raw_parameter_group in raw_parameter_groups: + name, resource = self._parse_parameter_group(raw_parameter_group) + self[name] = resource + + def _parse_parameter_group(self, raw_parameter_group): + raw_parameter_group['name'] = raw_parameter_group.pop('CacheParameterGroupName') + return raw_parameter_group['name'], raw_parameter_group \ No newline at end of file From 199424ce3320e97b75a6d396def0e8afcdb97d26 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 11 Mar 2019 17:29:55 -0400 Subject: [PATCH 342/667] Implemented ElastiCache security groups --- .../aws/resources/elasticache/securitygroups.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 ScoutSuite/providers/aws/resources/elasticache/securitygroups.py diff --git a/ScoutSuite/providers/aws/resources/elasticache/securitygroups.py b/ScoutSuite/providers/aws/resources/elasticache/securitygroups.py new file mode 100644 index 000000000..51cd26e59 --- /dev/null +++ b/ScoutSuite/providers/aws/resources/elasticache/securitygroups.py @@ -0,0 +1,14 @@ +from ScoutSuite.providers.aws.resources.resources import AWSResources + + +class SecurityGroups(AWSResources): + async def fetch_all(self, **kwargs): + raw_security_groups = await self.facade.elasticache.get_security_groups(self.scope['region']) + + for raw_security_group in raw_security_groups: + name, resource = self._parse_security_group(raw_security_group) + self[name] = resource + + def _parse_security_group(self, raw_security_group): + raw_security_group['name'] = raw_security_group.pop('CacheSecurityGroupName') + return raw_security_group['name'], raw_security_group From 5a5f6bfe69f316c47069212b3b001ab57175f0a9 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 11 Mar 2019 17:30:16 -0400 Subject: [PATCH 343/667] Implemented ElastiCache service and integrated it to AWS --- ScoutSuite/providers/aws/configs/services.py | 4 +-- .../aws/resources/elasticache/service.py | 29 +++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 ScoutSuite/providers/aws/resources/elasticache/service.py diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index 95acaa23c..16c1b1351 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -7,7 +7,7 @@ from ScoutSuite.providers.aws.resources.directconnect.service import DirectConnect from ScoutSuite.providers.aws.resources.ec2.service import EC2 from ScoutSuite.providers.aws.resources.efs.service import EFS -from ScoutSuite.providers.aws.services.elasticache import ElastiCacheConfig +from ScoutSuite.providers.aws.resources.elasticache.service import ElastiCache from ScoutSuite.providers.aws.services.elb import ELBConfig from ScoutSuite.providers.aws.services.elbv2 import ELBv2Config from ScoutSuite.providers.aws.services.emr import EMRConfig @@ -60,7 +60,7 @@ def __init__(self, metadata=None, thread_config=4, **kwargs): self.directconnect = DirectConnect() self.ec2 = EC2() self.efs = EFS() - self.elasticache = ElastiCacheConfig(metadata['database']['elasticache'], thread_config) + self.elasticache = ElastiCache() self.elb = ELBConfig(metadata['compute']['elb'], thread_config) self.elbv2 = ELBv2Config(metadata['compute']['elbv2'], thread_config) self.emr = EMRConfig(metadata['analytics']['emr'], thread_config) diff --git a/ScoutSuite/providers/aws/resources/elasticache/service.py b/ScoutSuite/providers/aws/resources/elasticache/service.py new file mode 100644 index 000000000..6e5dfee9f --- /dev/null +++ b/ScoutSuite/providers/aws/resources/elasticache/service.py @@ -0,0 +1,29 @@ +from ScoutSuite.providers.aws.facade.facade import AWSFacade +from ScoutSuite.providers.aws.resources.regions import Regions +from ScoutSuite.providers.aws.resources.elasticache.vpcs import ElastiCacheVpcs +from ScoutSuite.providers.aws.resources.elasticache.securitygroups import SecurityGroups +from ScoutSuite.providers.aws.resources.elasticache.parametergroups import ParameterGroups + + +class ElastiCache(Regions): + _children = [ + (ElastiCacheVpcs, 'vpcs'), + (SecurityGroups, 'security_groups'), + (ParameterGroups, 'parameter_groups') + ] + + def __init__(self): + super(ElastiCache, self).__init__('elasticache') + + async def fetch_all(self, credentials=None, regions=None, partition_name='aws'): + await super(ElastiCache, self).fetch_all(credentials, regions, partition_name) + + for region in self['regions']: + self['regions'][region]['clusters_count'] = sum([len(vpc['clusters']) for vpc in self['regions'][region]['vpcs'].values()]) + self['regions'][region]['subnet_groups_count'] = sum([len(vpc['subnet_groups']) for vpc in self['regions'][region]['vpcs'].values()]) + + self['clusters_count'] = sum([region['clusters_count'] for region in self['regions'].values()]) + + # We do not want the parameter groups to be part of the resources count, as it is usually in + # the three of four digits and would make the resources count confusing. + self.pop('parameter_groups_count') \ No newline at end of file From 7c3c31603a7a5bb2c1436321ad8d7c979420055c Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 11 Mar 2019 17:30:21 -0400 Subject: [PATCH 344/667] Removed previous implementation --- .../providers/aws/services/elasticache.py | 68 ------------------- 1 file changed, 68 deletions(-) delete mode 100644 ScoutSuite/providers/aws/services/elasticache.py diff --git a/ScoutSuite/providers/aws/services/elasticache.py b/ScoutSuite/providers/aws/services/elasticache.py deleted file mode 100644 index 092da241c..000000000 --- a/ScoutSuite/providers/aws/services/elasticache.py +++ /dev/null @@ -1,68 +0,0 @@ -# -*- coding: utf-8 -*- - -from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients -from ScoutSuite.providers.aws.configs.vpc import VPCConfig -from ScoutSuite.utils import manage_dictionary -from ScoutSuite.providers.aws.utils import ec2_classic - - -######################################## -# ElastiCacheRegionConfig -######################################## - -class ElastiCacheRegionConfig(RegionConfig): - """ - ElastiCache configuration for a single AWS region - """ - security_groups = {} - - def parse_cluster(self, global_params, region, cluster): - """ - Parse a single ElastiCache cluster - - :param global_params: Parameters shared for all regions - :param region: Name of the AWS region - :param cluster: ElastiCache cluster - """ - cluster_name = cluster.pop('CacheClusterId') - cluster['name'] = cluster_name - # Must fetch info about the subnet group to retrieve the VPC ID... - if 'CacheSubnetGroupName' in cluster: - subnet_group = \ - api_clients[region].describe_cache_subnet_groups(CacheSubnetGroupName=cluster['CacheSubnetGroupName'])[ - 'CacheSubnetGroups'][0] - vpc_id = subnet_group['VpcId'] - else: - vpc_id = ec2_classic - subnet_group = None - manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types)) - self.vpcs[vpc_id].clusters[cluster_name] = cluster - if subnet_group: - self.vpcs[vpc_id].subnet_groups[subnet_group['CacheSubnetGroupName']] = subnet_group - - def parse_security_group(self, global_params, region, security_group): - """ - Parse a single ElastiCache security group - - :param global_params: - :param region: - :param security_group: - :return: - """ - security_group['name'] = security_group.pop('CacheSecurityGroupName') - self.security_groups[security_group['name']] = security_group - - -######################################## -# ElastiCacheConfig -######################################## - -class ElastiCacheConfig(RegionalServiceConfig): - """ - ElastiCache configuration for all AWS regions - """ - - region_config_class = ElastiCacheRegionConfig - - def __init__(self, service_metadata, thread_config=4): - super(ElastiCacheConfig, self).__init__(service_metadata, thread_config) From 648ed299d0a96921932d188cc7169aa7555e0863 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 11 Mar 2019 17:48:34 -0400 Subject: [PATCH 345/667] Removed todo --- ScoutSuite/providers/aws/facade/elasticache.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/elasticache.py b/ScoutSuite/providers/aws/facade/elasticache.py index 5e3c592c3..6f3e07a9e 100644 --- a/ScoutSuite/providers/aws/facade/elasticache.py +++ b/ScoutSuite/providers/aws/facade/elasticache.py @@ -12,9 +12,7 @@ class ElastiCacheFacade(AWSBaseFacade): subnets_cache = {} async def get_clusters(self, region, vpc): - # This result could be cached to save some api calls await self.cache_clusters(region) - return [cluster for cluster in self.clusters_cache[region] if cluster['VpcId'] == vpc] async def cache_clusters(self, region): From 1e91971ad001c94fa67d279cd69319d9b4fe7bde Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 11 Mar 2019 17:58:02 -0400 Subject: [PATCH 346/667] Removed vpcs count --- ScoutSuite/providers/aws/resources/regions.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ScoutSuite/providers/aws/resources/regions.py b/ScoutSuite/providers/aws/resources/regions.py index 0d7c24ba4..d05227b1f 100644 --- a/ScoutSuite/providers/aws/resources/regions.py +++ b/ScoutSuite/providers/aws/resources/regions.py @@ -43,4 +43,9 @@ async def fetch_all(self, credentials, regions=None, partition_name='aws'): def _set_counts(self): self['regions_count'] = len(self['regions']) for _, key in self._children: + # VPCs should not be counted as resources. They exist whether you have resources or not, so + # counting them would make the report confusing. + if key == 'vpcs': + continue + self[key + '_count'] = sum([region[key + '_count'] for region in self['regions'].values()]) From ed5f31e288856ffc7b1692105b3e8ae50ff70ffc Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Mon, 11 Mar 2019 19:21:30 -0400 Subject: [PATCH 347/667] Working but ulgy incress/egress --- ScoutSuite/providers/aws/provider.py | 29 +++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index 3bb610562..b8b528090 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -106,6 +106,7 @@ def _add_security_group_name_to_ec2_grants(self): {'AWSAccountId': self.aws_account_id}) def _add_security_group_data_to_elbv2(self): + none = 'N/A' ec2_config = self.services['ec2'] elbv2_config = self.services['elbv2'] for region in elbv2_config['regions']: @@ -113,9 +114,31 @@ def _add_security_group_data_to_elbv2(self): for lb in elbv2_config['regions'][region]['vpcs'][vpc]['lbs']: for i in range(0, len(elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'])): for sg in ec2_config['regions'][region]['vpcs'][vpc]['security_groups']: - if elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][i]['GroupId'] == sg: - elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][i] = \ - ec2_config['regions'][region]['vpcs'][vpc]['security_groups'][sg] + try: + if elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][i][ + 'GroupId'] == sg: + elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][i] = \ + ec2_config['regions'][region]['vpcs'][vpc]['security_groups'][sg] + except KeyError: + pass + for protocol in elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][i][ + 'rules']['ingress']['protocols']: + for port in elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][i][ + 'rules']['ingress']['protocols'][protocol]['ports']: + elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][i][ + 'valid_inbound_rules'] = True + if port not in elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['listeners'] \ + and port != none: + elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][i][ + 'valid_inbound'] = False + for port in elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][i][ + 'rules']['egress']['protocols'][protocol]['ports']: + elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][i][ + 'valid_outbound_rules'] = True + if port not in elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['listeners'] \ + and port != none: + elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][i][ + 'valid_outbound_rules'] = False def _check_ec2_zone_distribution(self): regions = self.services['ec2']['regions'].values() From 86080d2b2d2f3d58390a66f7fa4d2a81f1f0b81c Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Mon, 11 Mar 2019 20:08:22 -0400 Subject: [PATCH 348/667] First part of ugly code refactor --- ScoutSuite/providers/aws/provider.py | 33 +++++++++++++--------------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index b8b528090..a11974ada 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -121,24 +121,21 @@ def _add_security_group_data_to_elbv2(self): ec2_config['regions'][region]['vpcs'][vpc]['security_groups'][sg] except KeyError: pass - for protocol in elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][i][ - 'rules']['ingress']['protocols']: - for port in elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][i][ - 'rules']['ingress']['protocols'][protocol]['ports']: - elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][i][ - 'valid_inbound_rules'] = True - if port not in elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['listeners'] \ - and port != none: - elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][i][ - 'valid_inbound'] = False - for port in elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][i][ - 'rules']['egress']['protocols'][protocol]['ports']: - elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][i][ - 'valid_outbound_rules'] = True - if port not in elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['listeners'] \ - and port != none: - elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][i][ - 'valid_outbound_rules'] = False + self.check_sg_rules(elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb], i, 'ingress') + self.check_sg_rules(elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb], i, 'egress') + + @staticmethod + def check_sg_rules(lb, index, traffic_type): + none = 'N/A' + if traffic_type == 'ingress': + output = 'valid_inbound_rules' + elif traffic_type == 'egress': + output = 'valid_outbound_rules' + for protocol in lb['security_groups'][index]['rules'][traffic_type]['protocols']: + for port in lb['security_groups'][index]['rules'][traffic_type]['protocols'][protocol]['ports']: + lb['security_groups'][index][output] = True + if port not in lb['listeners'] and port != none: + lb['security_groups'][index][output] = False def _check_ec2_zone_distribution(self): regions = self.services['ec2']['regions'].values() From 2bc6c3be90da2ceeffe79a763ca43152e976cfca Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Mon, 11 Mar 2019 20:15:55 -0400 Subject: [PATCH 349/667] Second part of ugly code refactor (moved func to service file) --- ScoutSuite/providers/aws/provider.py | 18 +++--------------- ScoutSuite/providers/aws/services/elbv2.py | 13 +++++++++++++ 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index a11974ada..e4473606a 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -8,6 +8,7 @@ from ScoutSuite.providers.aws.aws import get_aws_account_id from ScoutSuite.providers.aws.configs.services import AWSServicesConfig from ScoutSuite.providers.aws.services.vpc import put_cidr_name +from ScoutSuite.providers.aws.services.elbv2 import check_security_group_rules from ScoutSuite.providers.base.configs.browser import combine_paths, get_object_at, get_value_at from ScoutSuite.providers.base.provider import BaseProvider from ScoutSuite.utils import manage_dictionary @@ -121,21 +122,8 @@ def _add_security_group_data_to_elbv2(self): ec2_config['regions'][region]['vpcs'][vpc]['security_groups'][sg] except KeyError: pass - self.check_sg_rules(elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb], i, 'ingress') - self.check_sg_rules(elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb], i, 'egress') - - @staticmethod - def check_sg_rules(lb, index, traffic_type): - none = 'N/A' - if traffic_type == 'ingress': - output = 'valid_inbound_rules' - elif traffic_type == 'egress': - output = 'valid_outbound_rules' - for protocol in lb['security_groups'][index]['rules'][traffic_type]['protocols']: - for port in lb['security_groups'][index]['rules'][traffic_type]['protocols'][protocol]['ports']: - lb['security_groups'][index][output] = True - if port not in lb['listeners'] and port != none: - lb['security_groups'][index][output] = False + check_security_group_rules(elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb], i, 'ingress') + check_security_group_rules(elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb], i, 'egress') def _check_ec2_zone_distribution(self): regions = self.services['ec2']['regions'].values() diff --git a/ScoutSuite/providers/aws/services/elbv2.py b/ScoutSuite/providers/aws/services/elbv2.py index 74ee21aa0..2d489c6fb 100644 --- a/ScoutSuite/providers/aws/services/elbv2.py +++ b/ScoutSuite/providers/aws/services/elbv2.py @@ -79,3 +79,16 @@ class ELBv2Config(RegionalServiceConfig): def __init__(self, service_metadata, thread_config=4): super(ELBv2Config, self).__init__(service_metadata, thread_config) + + +def check_security_group_rules(lb, index, traffic_type): + none = 'N/A' + if traffic_type == 'ingress': + output = 'valid_inbound_rules' + elif traffic_type == 'egress': + output = 'valid_outbound_rules' + for protocol in lb['security_groups'][index]['rules'][traffic_type]['protocols']: + for port in lb['security_groups'][index]['rules'][traffic_type]['protocols'][protocol]['ports']: + lb['security_groups'][index][output] = True + if port not in lb['listeners'] and port != none: + lb['security_groups'][index][output] = False From 07dc10b5a1be077ddea9df25698d44a63e164f36 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 12 Mar 2019 10:34:14 -0400 Subject: [PATCH 350/667] Lint --- ScoutSuite/providers/aws/resources/cloudformation/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/resources/cloudformation/service.py b/ScoutSuite/providers/aws/resources/cloudformation/service.py index 3d21398c6..234db3475 100644 --- a/ScoutSuite/providers/aws/resources/cloudformation/service.py +++ b/ScoutSuite/providers/aws/resources/cloudformation/service.py @@ -19,7 +19,7 @@ def _parse_stack(self, raw_stack): template = raw_stack.pop('template') raw_stack['deletion_policy'] = self.has_deletion_policy(template) - + if hasattr(template, 'keys'): for group in template.keys(): if 'DeletionPolicy' in template[group]: From 1af73178e82229ad30d05218cce7636655713e1f Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Tue, 12 Mar 2019 19:00:44 -0400 Subject: [PATCH 351/667] Omae wa mou shindeiru'ed aws.py --- ScoutSuite/providers/aws/aws.py | 145 ------------------ ScoutSuite/providers/aws/configs/base.py | 2 +- ScoutSuite/providers/aws/configs/regions.py | 5 +- ScoutSuite/providers/aws/provider.py | 3 +- ScoutSuite/providers/aws/services/ec2.py | 3 +- ScoutSuite/providers/aws/services/efs.py | 2 +- ScoutSuite/providers/aws/services/elbv2.py | 4 +- ScoutSuite/providers/aws/services/iam.py | 3 +- ScoutSuite/providers/aws/services/rds.py | 3 +- ScoutSuite/providers/aws/services/redshift.py | 4 +- ScoutSuite/providers/aws/services/route53.py | 2 +- ScoutSuite/providers/aws/services/s3.py | 2 +- ScoutSuite/providers/aws/services/vpc.py | 3 +- ScoutSuite/providers/aws/utils.py | 142 ++++++++++++++++- ScoutSuite/providers/base/configs/base.py | 3 +- ScoutSuite/providers/base/configs/services.py | 2 +- 16 files changed, 152 insertions(+), 176 deletions(-) delete mode 100644 ScoutSuite/providers/aws/aws.py diff --git a/ScoutSuite/providers/aws/aws.py b/ScoutSuite/providers/aws/aws.py deleted file mode 100644 index 51db87a13..000000000 --- a/ScoutSuite/providers/aws/aws.py +++ /dev/null @@ -1,145 +0,0 @@ -# -*- coding: utf-8 -*- - -import time -from collections import Counter - -import boto3 -from botocore.session import Session - -from ScoutSuite.core.console import print_info, print_exception - - -def build_region_list(service, chosen_regions=None, partition_name='aws'): - """ - Build the list of target region names - - :param service: - :param chosen_regions: - :param partition_name: - - :return: - """ - if chosen_regions is None: - chosen_regions = [] - service = 'ec2containerservice' if service == 'ecs' else service - # Get list of regions from botocore - regions = Session().get_available_regions(service, partition_name=partition_name) - if len(chosen_regions): - return list((Counter(regions) & Counter(chosen_regions)).elements()) - else: - return regions - - -def connect_service(service, credentials, region_name=None, config=None, silent=False): - """ - Instantiates an AWS API client - - :param service: Service targeted, e.g. ec2 - :param credentials: Id, secret, token - :param region_name: Region desired, e.g. us-east-2 - :param config: Configuration (optional) - :param silent: Whether or not to print messages - - :return: - """ - api_client = None - try: - client_params = {'service_name': service.lower()} - session_params = {'aws_access_key_id': credentials.get('access_key'), - 'aws_secret_access_key': credentials.get('secret_key'), - 'aws_session_token': credentials.get('token')} - if region_name: - client_params['region_name'] = region_name - session_params['region_name'] = region_name - if config: - client_params['config'] = config - aws_session = boto3.session.Session(**session_params) - if not silent: - info_message = 'Connecting to AWS %s' % service - if region_name: - info_message = info_message + ' in %s' % region_name - print_info('%s...' % info_message) - api_client = aws_session.client(**client_params) - except Exception as e: - print_exception(e) - return api_client - - -def get_name(src, dst, default_attribute): - """ - - :param src: - :param dst: - :param default_attribute: - - :return: - """ - name_found = False - if 'Tags' in src: - for tag in src['Tags']: - if tag['Key'] == 'Name' and tag['Value'] != '': - dst['name'] = tag['Value'] - name_found = True - if not name_found: - dst['name'] = src[default_attribute] - return dst['name'] - - -def get_caller_identity(credentials): - api_client = connect_service('sts', credentials, silent=True) - return api_client.get_caller_identity() - - -def get_aws_account_id(credentials): - caller_identity = get_caller_identity(credentials) - return caller_identity['Arn'].split(':')[4] - - -def get_partition_name(credentials): - caller_identity = get_caller_identity(credentials) - return caller_identity['Arn'].split(':')[1] - - -def handle_truncated_response(callback, params, entities): - """ - Handle truncated responses - - :param callback: - :param params: - :param entities: - - :return: - """ - results = {} - for entity in entities: - results[entity] = [] - while True: - try: - marker_found = False - response = callback(**params) - for entity in entities: - if entity in response: - results[entity] = results[entity] + response[entity] - for marker_name in ['NextToken', 'Marker', 'PaginationToken']: - if marker_name in response and response[marker_name]: - params[marker_name] = response[marker_name] - marker_found = True - if not marker_found: - break - except Exception as e: - if is_throttled(e): - time.sleep(1) - else: - raise e - return results - - -def is_throttled(e): - """ - Determines whether the exception is due to API throttling. - - :param e: Exception raised - :return: True if it's a throttling exception else False - """ - return (hasattr(e, 'response') and 'Error' in e.response and e.response['Error']['Code'] in - ['Throttling', 'RequestLimitExceeded', 'ThrottlingException', 'TooManyRequestsException']) diff --git a/ScoutSuite/providers/aws/configs/base.py b/ScoutSuite/providers/aws/configs/base.py index 29d7c86f0..24d5d09ca 100644 --- a/ScoutSuite/providers/aws/configs/base.py +++ b/ScoutSuite/providers/aws/configs/base.py @@ -6,7 +6,7 @@ except ImportError: from queue import Queue -from ScoutSuite.providers.aws.aws import handle_truncated_response +from ScoutSuite.providers.aws.utils import handle_truncated_response from ScoutSuite.providers.base.configs.base import BaseConfig diff --git a/ScoutSuite/providers/aws/configs/regions.py b/ScoutSuite/providers/aws/configs/regions.py index e5e2e8ab7..254145635 100644 --- a/ScoutSuite/providers/aws/configs/regions.py +++ b/ScoutSuite/providers/aws/configs/regions.py @@ -13,15 +13,14 @@ except ImportError: from queue import Queue -from ScoutSuite.providers.aws.aws import build_region_list, connect_service, get_aws_account_id, get_name, \ - handle_truncated_response from ScoutSuite.core.console import print_exception, print_info from ScoutSuite.providers.base.configs import resource_id_map from ScoutSuite.providers.base.configs.threads import thread_configs from ScoutSuite.providers.aws.configs.vpc import VPCConfig from ScoutSuite.utils import format_service_name, manage_dictionary -from ScoutSuite.providers.aws.utils import is_throttled +from ScoutSuite.providers.aws.utils import is_throttled, build_region_list, connect_service, get_name, \ + get_aws_account_id, handle_truncated_response from ScoutSuite.providers.aws.configs.base import BaseConfig from ScoutSuite.output.console import FetchStatusLogger diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index a2f3412aa..58d3ee54e 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -5,13 +5,12 @@ import boto3 from ScoutSuite.core.console import print_debug, print_error, print_exception, print_info -from ScoutSuite.providers.aws.aws import get_aws_account_id from ScoutSuite.providers.aws.configs.services import AWSServicesConfig from ScoutSuite.providers.aws.services.vpc import put_cidr_name from ScoutSuite.providers.base.configs.browser import combine_paths, get_object_at, get_value_at from ScoutSuite.providers.base.provider import BaseProvider from ScoutSuite.utils import manage_dictionary -from ScoutSuite.providers.aws.utils import ec2_classic +from ScoutSuite.providers.aws.utils import ec2_classic, get_aws_account_id class AWSProvider(BaseProvider): diff --git a/ScoutSuite/providers/aws/services/ec2.py b/ScoutSuite/providers/aws/services/ec2.py index a4970881b..b9013fa83 100644 --- a/ScoutSuite/providers/aws/services/ec2.py +++ b/ScoutSuite/providers/aws/services/ec2.py @@ -9,12 +9,11 @@ import netaddr from ScoutSuite.core.console import print_exception, print_info -from ScoutSuite.providers.aws.aws import get_name from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients from ScoutSuite.providers.aws.configs.vpc import VPCConfig from ScoutSuite.providers.base.configs.browser import get_attribute_at from ScoutSuite.utils import manage_dictionary -from ScoutSuite.providers.aws.utils import ec2_classic, get_keys +from ScoutSuite.providers.aws.utils import ec2_classic, get_keys, get_name from ScoutSuite.core.fs import load_data ######################################## diff --git a/ScoutSuite/providers/aws/services/efs.py b/ScoutSuite/providers/aws/services/efs.py index ab4e6186a..5d6cf29dc 100644 --- a/ScoutSuite/providers/aws/services/efs.py +++ b/ScoutSuite/providers/aws/services/efs.py @@ -2,7 +2,7 @@ """ EFS-related classes and functions """ -from ScoutSuite.providers.aws.aws import handle_truncated_response +from ScoutSuite.providers.aws.utils import handle_truncated_response from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients diff --git a/ScoutSuite/providers/aws/services/elbv2.py b/ScoutSuite/providers/aws/services/elbv2.py index b8f93986b..775d1a564 100644 --- a/ScoutSuite/providers/aws/services/elbv2.py +++ b/ScoutSuite/providers/aws/services/elbv2.py @@ -3,12 +3,10 @@ ELBv2-related classes and functions """ -from ScoutSuite.providers.aws.aws import handle_truncated_response - from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients from ScoutSuite.providers.aws.configs.vpc import VPCConfig from ScoutSuite.utils import manage_dictionary -from ScoutSuite.providers.aws.utils import ec2_classic +from ScoutSuite.providers.aws.utils import ec2_classic, handle_truncated_response ######################################## diff --git a/ScoutSuite/providers/aws/services/iam.py b/ScoutSuite/providers/aws/services/iam.py index 48ee8d7af..1601ebf23 100644 --- a/ScoutSuite/providers/aws/services/iam.py +++ b/ScoutSuite/providers/aws/services/iam.py @@ -3,9 +3,8 @@ from botocore.exceptions import ClientError from ScoutSuite.core.console import print_error, print_exception -from ScoutSuite.providers.aws.aws import connect_service, handle_truncated_response from ScoutSuite.providers.aws.configs.base import AWSBaseConfig -from ScoutSuite.providers.aws.utils import get_keys, is_throttled +from ScoutSuite.providers.aws.utils import get_keys, is_throttled, connect_service, handle_truncated_response from ScoutSuite.utils import * diff --git a/ScoutSuite/providers/aws/services/rds.py b/ScoutSuite/providers/aws/services/rds.py index d563b1cf0..3cc375a26 100644 --- a/ScoutSuite/providers/aws/services/rds.py +++ b/ScoutSuite/providers/aws/services/rds.py @@ -1,11 +1,10 @@ # -*- coding: utf-8 -*- from ScoutSuite.core.console import print_error, print_exception -from ScoutSuite.providers.aws.aws import handle_truncated_response from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients from ScoutSuite.providers.aws.configs.vpc import VPCConfig from ScoutSuite.utils import manage_dictionary -from ScoutSuite.providers.aws.utils import ec2_classic +from ScoutSuite.providers.aws.utils import ec2_classic, handle_truncated_response ######################################## diff --git a/ScoutSuite/providers/aws/services/redshift.py b/ScoutSuite/providers/aws/services/redshift.py index 78e59482f..e51c6d503 100644 --- a/ScoutSuite/providers/aws/services/redshift.py +++ b/ScoutSuite/providers/aws/services/redshift.py @@ -3,12 +3,10 @@ Redshift-related classes and functions """ -from ScoutSuite.providers.aws.aws import handle_truncated_response - from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients from ScoutSuite.providers.aws.configs.vpc import VPCConfig from ScoutSuite.utils import manage_dictionary -from ScoutSuite.providers.aws.utils import ec2_classic +from ScoutSuite.providers.aws.utils import ec2_classic, handle_truncated_response ######################################## diff --git a/ScoutSuite/providers/aws/services/route53.py b/ScoutSuite/providers/aws/services/route53.py index 2cdbb1894..e35945425 100644 --- a/ScoutSuite/providers/aws/services/route53.py +++ b/ScoutSuite/providers/aws/services/route53.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from ScoutSuite.providers.aws.aws import handle_truncated_response +from ScoutSuite.providers.aws.utils import handle_truncated_response from ScoutSuite.providers.aws.configs.base import AWSBaseConfig diff --git a/ScoutSuite/providers/aws/services/s3.py b/ScoutSuite/providers/aws/services/s3.py index c44e207a3..3f0f4ec88 100644 --- a/ScoutSuite/providers/aws/services/s3.py +++ b/ScoutSuite/providers/aws/services/s3.py @@ -8,7 +8,7 @@ from botocore.exceptions import ClientError from ScoutSuite.core.console import print_error, print_exception, print_info -from ScoutSuite.providers.aws.aws import handle_truncated_response +from ScoutSuite.providers.aws.utils import handle_truncated_response from ScoutSuite.providers.aws.configs.base import AWSBaseConfig from ScoutSuite.utils import manage_dictionary diff --git a/ScoutSuite/providers/aws/services/vpc.py b/ScoutSuite/providers/aws/services/vpc.py index a5f2c1c58..2824350bb 100644 --- a/ScoutSuite/providers/aws/services/vpc.py +++ b/ScoutSuite/providers/aws/services/vpc.py @@ -4,12 +4,11 @@ import netaddr -from ScoutSuite.providers.aws.aws import get_name from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig from ScoutSuite.providers.aws.configs.vpc import VPCConfig as SingleVPCConfig from ScoutSuite.providers.base.configs.browser import get_value_at from ScoutSuite.utils import manage_dictionary -from ScoutSuite.providers.aws.utils import ec2_classic, get_keys +from ScoutSuite.providers.aws.utils import ec2_classic, get_keys, get_name from ScoutSuite.core.fs import load_data, read_ip_ranges ######################################## diff --git a/ScoutSuite/providers/aws/utils.py b/ScoutSuite/providers/aws/utils.py index fd4873a68..98afa7a28 100644 --- a/ScoutSuite/providers/aws/utils.py +++ b/ScoutSuite/providers/aws/utils.py @@ -1,10 +1,73 @@ # -*- coding: utf-8 -*- import re +import time +from collections.__init__ import Counter + +import boto3 +from botocore.session import Session + +from ScoutSuite.core.console import print_info, print_exception ec2_classic = 'EC2-Classic' +def build_region_list(service, chosen_regions=None, partition_name='aws'): + """ + Build the list of target region names + + :param service: + :param chosen_regions: + :param partition_name: + + :return: + """ + if chosen_regions is None: + chosen_regions = [] + service = 'ec2containerservice' if service == 'ecs' else service + # Get list of regions from botocore + regions = Session().get_available_regions(service, partition_name=partition_name) + if len(chosen_regions): + return list((Counter(regions) & Counter(chosen_regions)).elements()) + else: + return regions + + +def connect_service(service, credentials, region_name=None, config=None, silent=False): + """ + Instantiates an AWS API client + + :param service: Service targeted, e.g. ec2 + :param credentials: Id, secret, token + :param region_name: Region desired, e.g. us-east-2 + :param config: Configuration (optional) + :param silent: Whether or not to print messages + + :return: + """ + api_client = None + try: + client_params = {'service_name': service.lower()} + session_params = {'aws_access_key_id': credentials.get('access_key'), + 'aws_secret_access_key': credentials.get('secret_key'), + 'aws_session_token': credentials.get('token')} + if region_name: + client_params['region_name'] = region_name + session_params['region_name'] = region_name + if config: + client_params['config'] = config + aws_session = boto3.session.Session(**session_params) + if not silent: + info_message = 'Connecting to AWS %s' % service + if region_name: + info_message = info_message + ' in %s' % region_name + print_info('%s...' % info_message) + api_client = aws_session.client(**client_params) + except Exception as e: + print_exception(e) + return api_client + + def get_keys(src, dst, keys): """ Copies the value of keys from source object to dest object @@ -18,15 +81,73 @@ def get_keys(src, dst, keys): dst[key] = src[key] if key in src else None -def no_camel(name): +def get_name(src, dst, default_attribute): """ - Converts CamelCase to camel_case - :param name: + :param src: + :param dst: + :param default_attribute: + :return: """ - s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name) - return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() + name_found = False + if 'Tags' in src: + for tag in src['Tags']: + if tag['Key'] == 'Name' and tag['Value'] != '': + dst['name'] = tag['Value'] + name_found = True + if not name_found: + dst['name'] = src[default_attribute] + return dst['name'] + + +def get_caller_identity(credentials): + api_client = connect_service('sts', credentials, silent=True) + return api_client.get_caller_identity() + + +def get_aws_account_id(credentials): + caller_identity = get_caller_identity(credentials) + return caller_identity['Arn'].split(':')[4] + + +def get_partition_name(credentials): + caller_identity = get_caller_identity(credentials) + return caller_identity['Arn'].split(':')[1] + + +def handle_truncated_response(callback, params, entities): + """ + Handle truncated responses + + :param callback: + :param params: + :param entities: + + :return: + """ + results = {} + for entity in entities: + results[entity] = [] + while True: + try: + marker_found = False + response = callback(**params) + for entity in entities: + if entity in response: + results[entity] = results[entity] + response[entity] + for marker_name in ['NextToken', 'Marker', 'PaginationToken']: + if marker_name in response and response[marker_name]: + params[marker_name] = response[marker_name] + marker_found = True + if not marker_found: + break + except Exception as e: + if is_throttled(e): + time.sleep(1) + else: + raise e + return results def is_throttled(e): @@ -40,3 +161,14 @@ def is_throttled(e): and e.response['Error']['Code'] in ['Throttling', 'RequestLimitExceeded', 'ThrottlingException']) + + +def no_camel(name): + """ + Converts CamelCase to camel_case + + :param name: + :return: + """ + s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name) + return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() diff --git a/ScoutSuite/providers/base/configs/base.py b/ScoutSuite/providers/base/configs/base.py index 237fefc19..b4d445eac 100644 --- a/ScoutSuite/providers/base/configs/base.py +++ b/ScoutSuite/providers/base/configs/base.py @@ -16,11 +16,10 @@ from ScoutSuite.providers.base.configs.threads import thread_configs # TODO do this better without name conflict -from ScoutSuite.providers.aws.aws import connect_service from ScoutSuite.providers.gcp.utils import gcp_connect_service from ScoutSuite.providers.azure.utils import azure_connect_service -from ScoutSuite.providers.aws.aws import build_region_list +from ScoutSuite.providers.aws.utils import build_region_list, connect_service from ScoutSuite.core.console import print_exception, print_info from ScoutSuite.output.console import FetchStatusLogger diff --git a/ScoutSuite/providers/base/configs/services.py b/ScoutSuite/providers/base/configs/services.py index 0f8b951ce..806e17b6c 100644 --- a/ScoutSuite/providers/base/configs/services.py +++ b/ScoutSuite/providers/base/configs/services.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from ScoutSuite.core.console import print_error, print_exception, print_debug -from ScoutSuite.providers.aws.aws import get_partition_name +from ScoutSuite.providers.aws.utils import get_partition_name class BaseServicesConfig(object): From 02e8eab4fcfa759587965069974d2a0e96a52857 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Tue, 12 Mar 2019 19:06:32 -0400 Subject: [PATCH 352/667] Filled params in utils.py --- ScoutSuite/providers/aws/utils.py | 36 +++++++++++++++---------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/ScoutSuite/providers/aws/utils.py b/ScoutSuite/providers/aws/utils.py index 98afa7a28..d3351fa4a 100644 --- a/ScoutSuite/providers/aws/utils.py +++ b/ScoutSuite/providers/aws/utils.py @@ -16,9 +16,9 @@ def build_region_list(service, chosen_regions=None, partition_name='aws'): """ Build the list of target region names - :param service: - :param chosen_regions: - :param partition_name: + :param service: Service targeted, e.g. ec2 + :param chosen_regions: Regions desired, e.g. us-east-2 + :param partition_name: Name of the partition, default is aws :return: """ @@ -37,11 +37,11 @@ def connect_service(service, credentials, region_name=None, config=None, silent= """ Instantiates an AWS API client - :param service: Service targeted, e.g. ec2 - :param credentials: Id, secret, token - :param region_name: Region desired, e.g. us-east-2 - :param config: Configuration (optional) - :param silent: Whether or not to print messages + :param service: Service targeted, e.g. ec2 + :param credentials: Id, secret, token + :param region_name: Region desired, e.g. us-east-2 + :param config: Configuration (optional) + :param silent: Whether or not to print messages :return: """ @@ -72,9 +72,9 @@ def get_keys(src, dst, keys): """ Copies the value of keys from source object to dest object - :param src: - :param dst: - :param keys: + :param src: Source object + :param dst: Destination object + :param keys: Keys :return: """ for key in keys: @@ -84,9 +84,9 @@ def get_keys(src, dst, keys): def get_name(src, dst, default_attribute): """ - :param src: - :param dst: - :param default_attribute: + :param src: Source object + :param dst: Destination object + :param default_attribute: Default attribute :return: """ @@ -120,9 +120,9 @@ def handle_truncated_response(callback, params, entities): """ Handle truncated responses - :param callback: - :param params: - :param entities: + :param callback: Callback to process + :param params: Parameters to call callback with + :param entities: Entities :return: """ @@ -167,7 +167,7 @@ def no_camel(name): """ Converts CamelCase to camel_case - :param name: + :param name: Name string to convert :return: """ s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name) From 76dd49d95ea04f9aa75243c24f365795a23bf85f Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 12 Mar 2019 21:07:25 -0400 Subject: [PATCH 353/667] Implemented EMR facade --- ScoutSuite/providers/aws/facade/emr.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 ScoutSuite/providers/aws/facade/emr.py diff --git a/ScoutSuite/providers/aws/facade/emr.py b/ScoutSuite/providers/aws/facade/emr.py new file mode 100644 index 000000000..ec2f4bb77 --- /dev/null +++ b/ScoutSuite/providers/aws/facade/emr.py @@ -0,0 +1,15 @@ +from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils +from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade +from ScoutSuite.providers.utils import run_concurrently + + +class EMRFacade(AWSBaseFacade): + async def get_clusters(self, region): + clusters_list = await AWSFacadeUtils.get_all_pages('emr', region, self.session, 'list_clusters', 'Clusters') + client = AWSFacadeUtils.get_client('emr', region, self.session) + clusters_descriptions = [] + for cluster_id in [cluster['Id'] for cluster in clusters_list]: + cluster = run_concurrently(lambda: client.describe_cluster(ClusterId=cluster_id)['Cluster']) + clusters_descriptions.append(cluster) + + return cluster From 9a1d8bf73a5278d6a9f7e05a40605af219777382 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 12 Mar 2019 21:07:33 -0400 Subject: [PATCH 354/667] Integrated EMR facade to AWS facade --- ScoutSuite/providers/aws/facade/facade.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ScoutSuite/providers/aws/facade/facade.py b/ScoutSuite/providers/aws/facade/facade.py index e0feccb46..fcfb39c7c 100644 --- a/ScoutSuite/providers/aws/facade/facade.py +++ b/ScoutSuite/providers/aws/facade/facade.py @@ -10,6 +10,7 @@ from ScoutSuite.providers.aws.facade.ec2 import EC2Facade from ScoutSuite.providers.aws.facade.efs import EFSFacade from ScoutSuite.providers.aws.facade.elasticache import ElastiCacheFacade +from ScoutSuite.providers.aws.facade.emr import EMRFacade from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade from ScoutSuite.providers.utils import run_concurrently from ScoutSuite.core.console import print_error, print_debug @@ -27,6 +28,8 @@ def __init__(self, credentials: dict = None): self.directconnect = DirectConnectFacade(self.session) self.efs = EFSFacade(self.session) self.elasticache = ElastiCacheFacade(self.session) + self.emr = EMRFacade(self.session) + self.emr = EMRFacade(self.session) async def build_region_list(self, service: str, chosen_regions=None, partition_name='aws'): service = 'ec2containerservice' if service == 'ecs' else service @@ -64,3 +67,4 @@ def _set_session(self, credentials: dict): self.directconnect = DirectConnectFacade(self.session) self.efs = EFSFacade(self.session) self.elasticache = ElastiCacheFacade(self.session) + self.emr = EMRFacade(self.session) From f1b64d3935106c65247f5b09a9e277f45fdf9457 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 12 Mar 2019 21:12:55 -0400 Subject: [PATCH 355/667] Started migrating EMR --- .../providers/aws/resources/emr/service.py | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 ScoutSuite/providers/aws/resources/emr/service.py diff --git a/ScoutSuite/providers/aws/resources/emr/service.py b/ScoutSuite/providers/aws/resources/emr/service.py new file mode 100644 index 000000000..c83d7d0b5 --- /dev/null +++ b/ScoutSuite/providers/aws/resources/emr/service.py @@ -0,0 +1,84 @@ +from ScoutSuite.providers.aws.facade.facade import AWSFacade +from ScoutSuite.providers.aws.resources.regions import Regions +from ScoutSuite.providers.aws.resources.vpcs import Vpcs +from ScoutSuite.providers.aws.resources.resources import AWSResources + + +class EMRClusters(AWSResources): + async def fetch_all(self, **kwargs): + raw_clusters = await self.facade.emr.get_clusters(self.scope['region']) + for raw_cluster in raw_clusters: + name, resource = self._parse_cluster(raw_cluster) + self[name] = resource + + def _parse_cluster(self, raw_cluster): + raw_cluster['id'] = raw_cluster.pop('Id') + raw_cluster['name'] = raw_cluster.pop('Name') + vpc_id = 'TODO' # The EMR API won't disclose the VPC ID, so wait until all configs have been fetch and look + # # up the VPC based on the subnet ID + # manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types)) + return raw_cluster['id'], raw_cluster + + +class EMRVpcs(Vpcs): + children: [ + (EMRClusters, 'clusters') + ] + + +class EMR(Regions): + _children = [ + (EMRVpcs, 'vpcs') + ] + + def __init__(self): + super(EMR, self).__init__('emr') + + +# # -*- coding: utf-8 -*- + +# from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients +# from ScoutSuite.providers.aws.configs.vpc import VPCConfig +# from ScoutSuite.utils import manage_dictionary + + +# ######################################## +# # EMRRegionConfig +# ######################################## + +# class EMRRegionConfig(RegionConfig): +# """ +# EMR configuration for a single AWS region +# """ + +# def parse_cluster(self, global_params, region, cluster): +# """ +# Parse a single EMR cluster + +# :param global_params: Parameters shared for all regions +# :param region: Name of the AWS region +# :param cluster: EMR cluster +# """ +# cluster_id = cluster['Id'] +# cluster = api_clients[region].describe_cluster(ClusterId=cluster_id)['Cluster'] +# cluster['id'] = cluster.pop('Id') +# cluster['name'] = cluster.pop('Name') +# vpc_id = 'TODO' # The EMR API won't disclose the VPC ID, so wait until all configs have been fetch and look +# # up the VPC based on the subnet ID +# manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types)) +# self.vpcs[vpc_id].clusters[cluster_id] = cluster + + +# ######################################## +# # EMRConfig +# ######################################## + +# class EMRConfig(RegionalServiceConfig): +# """ +# EMR configuration for all AWS regions +# """ + +# region_config_class = EMRRegionConfig + +# def __init__(self, service_metadata, thread_config=4): +# super(EMRConfig, self).__init__(service_metadata, thread_config) From bcc1c72121068d4db0cad81ae61d5a90c5356d01 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 12 Mar 2019 21:48:43 -0400 Subject: [PATCH 356/667] Fixed wrong variable name --- ScoutSuite/providers/aws/facade/emr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/emr.py b/ScoutSuite/providers/aws/facade/emr.py index ec2f4bb77..18280a789 100644 --- a/ScoutSuite/providers/aws/facade/emr.py +++ b/ScoutSuite/providers/aws/facade/emr.py @@ -9,7 +9,7 @@ async def get_clusters(self, region): client = AWSFacadeUtils.get_client('emr', region, self.session) clusters_descriptions = [] for cluster_id in [cluster['Id'] for cluster in clusters_list]: - cluster = run_concurrently(lambda: client.describe_cluster(ClusterId=cluster_id)['Cluster']) + cluster = client.describe_cluster(ClusterId=cluster_id)['Cluster'] clusters_descriptions.append(cluster) - return cluster + return clusters_descriptions From ffacdbd318fb284833b3836fe89b06df57ea1b4a Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 12 Mar 2019 21:49:08 -0400 Subject: [PATCH 357/667] Removed comment --- ScoutSuite/providers/aws/resources/vpcs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ScoutSuite/providers/aws/resources/vpcs.py b/ScoutSuite/providers/aws/resources/vpcs.py index d4bf9d879..6443c6466 100644 --- a/ScoutSuite/providers/aws/resources/vpcs.py +++ b/ScoutSuite/providers/aws/resources/vpcs.py @@ -19,7 +19,6 @@ async def fetch_all(self, **kwargs): name, resource = self._parse_vpc(vpc) self[name] = resource - # TODO: make a refactoring of the following: if len(self) == 0: return From eec34e7ddd82de1fedc2da0bf2d052ebe9c38a64 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 12 Mar 2019 21:49:18 -0400 Subject: [PATCH 358/667] Fixed migration --- .../providers/aws/resources/emr/service.py | 70 +++++-------------- 1 file changed, 16 insertions(+), 54 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/emr/service.py b/ScoutSuite/providers/aws/resources/emr/service.py index c83d7d0b5..084245cc0 100644 --- a/ScoutSuite/providers/aws/resources/emr/service.py +++ b/ScoutSuite/providers/aws/resources/emr/service.py @@ -1,7 +1,6 @@ from ScoutSuite.providers.aws.facade.facade import AWSFacade from ScoutSuite.providers.aws.resources.regions import Regions -from ScoutSuite.providers.aws.resources.vpcs import Vpcs -from ScoutSuite.providers.aws.resources.resources import AWSResources +from ScoutSuite.providers.aws.resources.resources import AWSResources, AWSCompositeResources class EMRClusters(AWSResources): @@ -14,17 +13,21 @@ async def fetch_all(self, **kwargs): def _parse_cluster(self, raw_cluster): raw_cluster['id'] = raw_cluster.pop('Id') raw_cluster['name'] = raw_cluster.pop('Name') - vpc_id = 'TODO' # The EMR API won't disclose the VPC ID, so wait until all configs have been fetch and look - # # up the VPC based on the subnet ID - # manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types)) return raw_cluster['id'], raw_cluster -class EMRVpcs(Vpcs): - children: [ +class EMRVpcs(AWSCompositeResources): + _children = [ (EMRClusters, 'clusters') ] + async def fetch_all(self, **kwargs): + # EMR won't disclose its VPC, so we put everything in a VPC named "TODO", and we + # infer the VPC afterwards during the preprocessing. + tmp_vpc = 'TODO' + self[tmp_vpc] = {} + await self._fetch_children(self[tmp_vpc], {'region': self.scope['region'], 'vpc': tmp_vpc}) + class EMR(Regions): _children = [ @@ -34,51 +37,10 @@ class EMR(Regions): def __init__(self): super(EMR, self).__init__('emr') + async def fetch_all(self, credentials=None, regions=None, partition_name='aws'): + await super(EMR, self).fetch_all(credentials, regions, partition_name) -# # -*- coding: utf-8 -*- - -# from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients -# from ScoutSuite.providers.aws.configs.vpc import VPCConfig -# from ScoutSuite.utils import manage_dictionary - - -# ######################################## -# # EMRRegionConfig -# ######################################## - -# class EMRRegionConfig(RegionConfig): -# """ -# EMR configuration for a single AWS region -# """ - -# def parse_cluster(self, global_params, region, cluster): -# """ -# Parse a single EMR cluster - -# :param global_params: Parameters shared for all regions -# :param region: Name of the AWS region -# :param cluster: EMR cluster -# """ -# cluster_id = cluster['Id'] -# cluster = api_clients[region].describe_cluster(ClusterId=cluster_id)['Cluster'] -# cluster['id'] = cluster.pop('Id') -# cluster['name'] = cluster.pop('Name') -# vpc_id = 'TODO' # The EMR API won't disclose the VPC ID, so wait until all configs have been fetch and look -# # up the VPC based on the subnet ID -# manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types)) -# self.vpcs[vpc_id].clusters[cluster_id] = cluster - - -# ######################################## -# # EMRConfig -# ######################################## - -# class EMRConfig(RegionalServiceConfig): -# """ -# EMR configuration for all AWS regions -# """ - -# region_config_class = EMRRegionConfig - -# def __init__(self, service_metadata, thread_config=4): -# super(EMRConfig, self).__init__(service_metadata, thread_config) + for region in self['regions']: + self['regions'][region]['clusters_count'] = sum([len(vpc['clusters']) for vpc in self['regions'][region]['vpcs'].values()]) + + self['clusters_count'] = sum([region['clusters_count'] for region in self['regions'].values()]) From fb29f7c335b75cf548003bc97f66c00d61084aaa Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 12 Mar 2019 21:49:56 -0400 Subject: [PATCH 359/667] Fixed region being hardcoded --- ScoutSuite/providers/aws/provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index 28dbac042..cbeae3005 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -492,7 +492,7 @@ def _set_emr_vpc_ids(self): self.set_emr_vpc_ids_callback, {'clear_list': clear_list}) for region in clear_list: - self.services['emr']['regions']['region']['vpcs'].pop('TODO') + self.services['emr']['regions'][region]['vpcs'].pop('TODO') def set_emr_vpc_ids_callback(self, current_config, path, current_path, vpc_id, callback_args): if vpc_id != 'TODO': From d24c6fed4b03279e11f8396087261ed9739db28e Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 12 Mar 2019 21:50:02 -0400 Subject: [PATCH 360/667] Integrated migration --- ScoutSuite/providers/aws/configs/services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index 16c1b1351..2c5311aef 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -10,7 +10,7 @@ from ScoutSuite.providers.aws.resources.elasticache.service import ElastiCache from ScoutSuite.providers.aws.services.elb import ELBConfig from ScoutSuite.providers.aws.services.elbv2 import ELBv2Config -from ScoutSuite.providers.aws.services.emr import EMRConfig +from ScoutSuite.providers.aws.resources.emr.service import EMR from ScoutSuite.providers.aws.services.iam import IAMConfig from ScoutSuite.providers.aws.services.rds import RDSConfig from ScoutSuite.providers.aws.services.redshift import RedshiftConfig @@ -63,7 +63,7 @@ def __init__(self, metadata=None, thread_config=4, **kwargs): self.elasticache = ElastiCache() self.elb = ELBConfig(metadata['compute']['elb'], thread_config) self.elbv2 = ELBv2Config(metadata['compute']['elbv2'], thread_config) - self.emr = EMRConfig(metadata['analytics']['emr'], thread_config) + self.emr = EMR() self.iam = IAMConfig(thread_config) self.awslambda = Lambdas() self.redshift = RedshiftConfig(metadata['database']['redshift'], thread_config) From 574cba138d5999814c12481f3d317137a4bbd3d0 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 12 Mar 2019 22:11:00 -0400 Subject: [PATCH 361/667] Removed old implementation --- ScoutSuite/providers/aws/services/emr.py | 47 ------------------------ 1 file changed, 47 deletions(-) delete mode 100644 ScoutSuite/providers/aws/services/emr.py diff --git a/ScoutSuite/providers/aws/services/emr.py b/ScoutSuite/providers/aws/services/emr.py deleted file mode 100644 index 814cbfdc3..000000000 --- a/ScoutSuite/providers/aws/services/emr.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- - -from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients -from ScoutSuite.providers.aws.configs.vpc import VPCConfig -from ScoutSuite.utils import manage_dictionary - - -######################################## -# EMRRegionConfig -######################################## - -class EMRRegionConfig(RegionConfig): - """ - EMR configuration for a single AWS region - """ - - def parse_cluster(self, global_params, region, cluster): - """ - Parse a single EMR cluster - - :param global_params: Parameters shared for all regions - :param region: Name of the AWS region - :param cluster: EMR cluster - """ - cluster_id = cluster['Id'] - cluster = api_clients[region].describe_cluster(ClusterId=cluster_id)['Cluster'] - cluster['id'] = cluster.pop('Id') - cluster['name'] = cluster.pop('Name') - vpc_id = 'TODO' # The EMR API won't disclose the VPC ID, so wait until all configs have been fetch and look - # up the VPC based on the subnet ID - manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types)) - self.vpcs[vpc_id].clusters[cluster_id] = cluster - - -######################################## -# EMRConfig -######################################## - -class EMRConfig(RegionalServiceConfig): - """ - EMR configuration for all AWS regions - """ - - region_config_class = EMRRegionConfig - - def __init__(self, service_metadata, thread_config=4): - super(EMRConfig, self).__init__(service_metadata, thread_config) From 2cd55ce5abde12ecc93326c7c9c78b36650d57a6 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Wed, 13 Mar 2019 07:56:55 -0400 Subject: [PATCH 362/667] Deleted profiles.py and the block of cause that used it, will create an issue to replace that and implement it properly. --- ScoutSuite/__main__.py | 30 +------ ScoutSuite/providers/aws/profiles.py | 126 --------------------------- 2 files changed, 4 insertions(+), 152 deletions(-) delete mode 100644 ScoutSuite/providers/aws/profiles.py diff --git a/ScoutSuite/__main__.py b/ScoutSuite/__main__.py index baf77e904..9aa8f6ee8 100644 --- a/ScoutSuite/__main__.py +++ b/ScoutSuite/__main__.py @@ -2,19 +2,16 @@ # -*- coding: utf-8 -*- import copy -import json import os import webbrowser -from ScoutSuite.core.console import config_debug_level, print_info, print_debug -from ScoutSuite.providers.aws.profiles import AWSProfiles - -from ScoutSuite.core.cli_parser import ScoutSuiteArgumentParser from ScoutSuite import AWSCONFIG -from ScoutSuite.output.html import Scout2Report +from ScoutSuite.core.cli_parser import ScoutSuiteArgumentParser +from ScoutSuite.core.console import config_debug_level, print_info, print_debug from ScoutSuite.core.exceptions import RuleExceptions -from ScoutSuite.core.ruleset import Ruleset from ScoutSuite.core.processingengine import ProcessingEngine +from ScoutSuite.core.ruleset import Ruleset +from ScoutSuite.output.html import Scout2Report from ScoutSuite.providers import get_provider @@ -128,25 +125,6 @@ def main(args=None): # Finalize cloud_provider.postprocessing(report.current_time, finding_rules) - # TODO: this is AWS-specific - move to postprocessing? - # This is partially implemented - # Get organization data if it exists - try: - profile = AWSProfiles.get(args.get('profile'))[0] - if 'source_profile' in profile.attributes: - organization_info_file = os.path.join(os.path.expanduser('~/.aws/recipes/%s/organization.json' % - profile.attributes['source_profile'])) - if os.path.isfile(organization_info_file): - with open(organization_info_file, 'rt') as f: - org = {} - accounts = json.load(f) - for account in accounts: - account_id = account.pop('Id') - org[account_id] = account - setattr(cloud_provider, 'organization', org) - except Exception as e: - pass - # Save config and create HTML report html_report_path = report.save(cloud_provider, exceptions, args.get('force_write'), args.get('debug')) diff --git a/ScoutSuite/providers/aws/profiles.py b/ScoutSuite/providers/aws/profiles.py deleted file mode 100644 index f32da4e46..000000000 --- a/ScoutSuite/providers/aws/profiles.py +++ /dev/null @@ -1,126 +0,0 @@ -# -*- coding: utf-8 -*- - -import os -import re - -from ScoutSuite.core.console import print_debug - -aws_dir = os.path.join(os.path.expanduser('~'), '.aws') -aws_credentials_file = os.path.join(aws_dir, 'credentials') -aws_config_file = os.path.join(aws_dir, 'config') - -re_profile_name = re.compile(r'(\[(profile\s+)?(.*?)\])') - - -class AWSProfile(object): - - def __init__(self, filename=None, raw_profile=None, name=None, credentials=None, account_id=None): - self.credentials = credentials - self.filename = filename - self.raw_profile = raw_profile - self.name = name - self.account_id = account_id - self.attributes = {} - if self.raw_profile: - self.parse_raw_profile() - - def parse_raw_profile(self): - for line in self.raw_profile.split('\n')[1:]: - line = line.strip() - if line: - values = line.split('=') - attribute = values[0].strip() - value = ''.join(values[1:]).strip() - self.attributes[attribute] = value - - def write(self): - tmp = AWSProfiles.get(self.name, quiet=True) - if not self.raw_profile: - self.raw_profile = tmp[0].raw_profile if len(tmp) else None - if not self.filename: - self.filename = tmp[0].filename if len(tmp) else self.filename - if not self.raw_profile: - if 'role_arn' in self.attributes and 'source_profile' in self.attributes: - self.filename = aws_config_file - new_raw_profile = '\n[profile %s]' % self.name - else: - self.filename = aws_credentials_file - new_raw_profile = '\n[%s]' % self.name - for attribute in self.attributes: - new_raw_profile += '\n%s=%s' % (attribute, self.attributes[attribute]) - with open(self.filename, 'a') as f: - f.write(new_raw_profile) - else: - new_raw_profile = '' - for line in self.raw_profile.splitlines(): - line_updated = False - for attribute in self.attributes: - if line.startswith(attribute): - new_raw_profile += '%s=%s\n' % (attribute, self.attributes[attribute]) - line_updated = True - break - if not line_updated: - new_raw_profile += '%s\n' % line - with open(self.filename, 'rt') as f: - contents = f.read() - contents = contents.replace(self.raw_profile, new_raw_profile) - with open(self.filename, 'wt') as f: - f.write(contents) - - -class AWSProfiles(object): - - @staticmethod - def list(names=None): - """ - @brief - - :return: List of all profile names found in .aws/config and .aws/credentials - """ - if names is None: - names = [] - return [p.name for p in AWSProfiles.get(names)] - - @staticmethod - def get(names=None, quiet=False): - """ - """ - if names is None: - names = [] - profiles = [] - profiles += AWSProfiles.find_profiles_in_file(aws_credentials_file, names, quiet) - profiles += AWSProfiles.find_profiles_in_file(aws_config_file, names, quiet) - return profiles - - @staticmethod - def find_profiles_in_file(filename, names=None, quiet=True): - if names is None: - names = [] - profiles = [] - if type(names) != list: - names = [names] - if not quiet: - print_debug('Searching for profiles matching %s in %s ... ' % (str(names), filename)) - name_filters = [] - for name in names: - name_filters.append(re.compile('^%s$' % name)) - if os.path.isfile(filename): - with open(filename, 'rt') as f: - aws_credentials = f.read() - existing_profiles = re_profile_name.findall(aws_credentials) - profile_count = len(existing_profiles) - 1 - for i, profile in enumerate(existing_profiles): - matching_profile = False - raw_profile = None - for name_filter in name_filters: - if name_filter.match(profile[2]): - matching_profile = True - i1 = aws_credentials.index(profile[0]) - if i < profile_count: - i2 = aws_credentials.index(existing_profiles[i + 1][0]) - raw_profile = aws_credentials[i1:i2] - else: - raw_profile = aws_credentials[i1:] - if len(name_filters) == 0 or matching_profile: - profiles.append(AWSProfile(filename=filename, raw_profile=raw_profile, name=profile[2])) - return profiles From 417c9be5cbf09c9aef907393aaf2defb61b35ab2 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Wed, 13 Mar 2019 08:23:09 -0400 Subject: [PATCH 363/667] Python 2.7 tests fix attemp --- ScoutSuite/providers/aws/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/utils.py b/ScoutSuite/providers/aws/utils.py index d3351fa4a..716771ee7 100644 --- a/ScoutSuite/providers/aws/utils.py +++ b/ScoutSuite/providers/aws/utils.py @@ -2,7 +2,7 @@ import re import time -from collections.__init__ import Counter +from collections import Counter import boto3 from botocore.session import Session From 409bad9c9818a7344640adc959ba022cba87daa5 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 13 Mar 2019 08:58:51 -0400 Subject: [PATCH 364/667] Removed duplicated assignation --- ScoutSuite/providers/aws/facade/facade.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ScoutSuite/providers/aws/facade/facade.py b/ScoutSuite/providers/aws/facade/facade.py index fcfb39c7c..92d09fa63 100644 --- a/ScoutSuite/providers/aws/facade/facade.py +++ b/ScoutSuite/providers/aws/facade/facade.py @@ -29,7 +29,6 @@ def __init__(self, credentials: dict = None): self.efs = EFSFacade(self.session) self.elasticache = ElastiCacheFacade(self.session) self.emr = EMRFacade(self.session) - self.emr = EMRFacade(self.session) async def build_region_list(self, service: str, chosen_regions=None, partition_name='aws'): service = 'ec2containerservice' if service == 'ecs' else service From 66dfdbfa6bbe0f1f9a4a8283fa2171387c500829 Mon Sep 17 00:00:00 2001 From: misg Date: Wed, 13 Mar 2019 13:14:15 -0400 Subject: [PATCH 365/667] Remove alias. --- ScoutSuite/providers/azure/configs/services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/azure/configs/services.py b/ScoutSuite/providers/azure/configs/services.py index 7876e23c5..15b283b74 100644 --- a/ScoutSuite/providers/azure/configs/services.py +++ b/ScoutSuite/providers/azure/configs/services.py @@ -4,7 +4,7 @@ from ScoutSuite.providers.azure.services.storageaccounts import StorageAccountsConfig from ScoutSuite.providers.azure.services.monitor import MonitorConfig from ScoutSuite.providers.azure.resources.sqldatabase.servers import Servers as SQLDatabaseConfig -from ScoutSuite.providers.azure.resources.securitycenter.security_center import SecurityCenter as SecurityCenterConfig +from ScoutSuite.providers.azure.resources.securitycenter.security_center import SecurityCenter from ScoutSuite.providers.azure.services.network import NetworkConfig from ScoutSuite.providers.azure.services.keyvault import KeyVaultConfig try: @@ -32,7 +32,7 @@ def __init__(self, metadata=None, thread_config=4, **kwargs): self.storageaccounts = StorageAccountsConfig(thread_config=thread_config) self.monitor = MonitorConfig(thread_config=thread_config) self.sqldatabase = SQLDatabaseConfig() - self.securitycenter = SecurityCenterConfig() + self.securitycenter = SecurityCenter() self.network = NetworkConfig(thread_config=thread_config) self.keyvault = KeyVaultConfig(thread_config=thread_config) From 79ad841c6b484dc12ad20e5b6f4cda954f3f42e0 Mon Sep 17 00:00:00 2001 From: misg Date: Wed, 13 Mar 2019 13:18:29 -0400 Subject: [PATCH 366/667] Rename _parse methods. --- .../securitycenter/auto_provisioning_settings.py | 11 +++++------ .../azure/resources/securitycenter/pricings.py | 4 ++-- .../resources/securitycenter/security_contacts.py | 4 ++-- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/securitycenter/auto_provisioning_settings.py b/ScoutSuite/providers/azure/resources/securitycenter/auto_provisioning_settings.py index 19a5919b4..bacb472c0 100644 --- a/ScoutSuite/providers/azure/resources/securitycenter/auto_provisioning_settings.py +++ b/ScoutSuite/providers/azure/resources/securitycenter/auto_provisioning_settings.py @@ -2,19 +2,18 @@ class AutoProvisioningSettings(Resources): - def __init__(self, facade): self.facade = facade async def fetch_all(self): for raw_settings in await self.facade.get_auto_provisioning_settings(): - id, auto_provisioning_settings = self._parse(raw_settings) + id, auto_provisioning_settings = self._parse_auto_provisioning_settings(raw_settings) self[id] = auto_provisioning_settings - def _parse(self, auto_provisioning_setting): + def _parse_auto_provisioning_settings(self, auto_provisioning_settings): auto_provisioning_setting_dict = {} - auto_provisioning_setting_dict['id'] = auto_provisioning_setting.id - auto_provisioning_setting_dict['name'] = auto_provisioning_setting.name - auto_provisioning_setting_dict['auto_provision'] = auto_provisioning_setting.auto_provision + auto_provisioning_setting_dict['id'] = auto_provisioning_settings.id + auto_provisioning_setting_dict['name'] = auto_provisioning_settings.name + auto_provisioning_setting_dict['auto_provision'] = auto_provisioning_settings.auto_provision return auto_provisioning_setting_dict['id'], auto_provisioning_setting_dict diff --git a/ScoutSuite/providers/azure/resources/securitycenter/pricings.py b/ScoutSuite/providers/azure/resources/securitycenter/pricings.py index d87bfe2b8..48fc7253e 100644 --- a/ScoutSuite/providers/azure/resources/securitycenter/pricings.py +++ b/ScoutSuite/providers/azure/resources/securitycenter/pricings.py @@ -8,10 +8,10 @@ def __init__(self, facade): async def fetch_all(self): for raw_pricing in await self.facade.get_pricings(): - id, pricing = self._parse(raw_pricing) + id, pricing = self._parse_pricing(raw_pricing) self[id] = pricing - def _parse(self, pricing): + def _parse_pricing(self, pricing): pricing_dict = {} pricing_dict['id'] = pricing.id pricing_dict['name'] = pricing.name diff --git a/ScoutSuite/providers/azure/resources/securitycenter/security_contacts.py b/ScoutSuite/providers/azure/resources/securitycenter/security_contacts.py index a7f9b23ea..c52b0dfcf 100644 --- a/ScoutSuite/providers/azure/resources/securitycenter/security_contacts.py +++ b/ScoutSuite/providers/azure/resources/securitycenter/security_contacts.py @@ -8,10 +8,10 @@ def __init__(self, facade): async def fetch_all(self): for raw_contact in await self.facade.get_security_contacts(): - id, security_contact = self._parse(raw_contact) + id, security_contact = self._parse_security_contact(raw_contact) self[id] = security_contact - def _parse(self, security_contact): + def _parse_security_contact(self, security_contact): security_contact_dict = {} security_contact_dict['id'] = security_contact.id security_contact_dict['name'] = security_contact.name From 321850dc11713982a758bc49bbd8a5a9fc0a26aa Mon Sep 17 00:00:00 2001 From: misg Date: Wed, 13 Mar 2019 13:24:24 -0400 Subject: [PATCH 367/667] Rename _parse methods. --- .../azure/resources/network/network_security_groups.py | 5 ++--- .../providers/azure/resources/network/network_watchers.py | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/network/network_security_groups.py b/ScoutSuite/providers/azure/resources/network/network_security_groups.py index bf7117450..6e65b65a0 100644 --- a/ScoutSuite/providers/azure/resources/network/network_security_groups.py +++ b/ScoutSuite/providers/azure/resources/network/network_security_groups.py @@ -2,16 +2,15 @@ class NetworkSecurityGroups(Resources): - def __init__(self, facade): self.facade = facade async def fetch_all(self): for raw_group in await self.facade.get_network_security_groups(): - id, network_security_group = self._parse(raw_group) + id, network_security_group = self._parse_network_security_group(raw_group) self[id] = network_security_group - def _parse(self, network_security_group): + def _parse_network_security_group(self, network_security_group): network_security_group_dict = {} network_security_group_dict['id'] = network_security_group.id network_security_group_dict['name'] = network_security_group.name diff --git a/ScoutSuite/providers/azure/resources/network/network_watchers.py b/ScoutSuite/providers/azure/resources/network/network_watchers.py index 7548cf667..0121f2f1b 100644 --- a/ScoutSuite/providers/azure/resources/network/network_watchers.py +++ b/ScoutSuite/providers/azure/resources/network/network_watchers.py @@ -2,16 +2,15 @@ class NetworkWatchers(Resources): - def __init__(self, facade): self.facade = facade async def fetch_all(self): for raw_watcher in await self.facade.get_network_watchers(): - id, network_watcher = self._parse(raw_watcher) + id, network_watcher = self._parse_network_watcher(raw_watcher) self[id] = network_watcher - def _parse(self, network_watcher): + def _parse_network_watcher(self, network_watcher): network_watcher_dict = {} network_watcher_dict['id'] = network_watcher.id network_watcher_dict['name'] = network_watcher.name From f6709ebe1fe07b0110ce23dd7cdf038a4ebaa424 Mon Sep 17 00:00:00 2001 From: misg Date: Wed, 13 Mar 2019 13:28:53 -0400 Subject: [PATCH 368/667] Rename _parse method. --- ScoutSuite/providers/azure/resources/keyvault/key_vaults.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/keyvault/key_vaults.py b/ScoutSuite/providers/azure/resources/keyvault/key_vaults.py index 39f843005..3c6d3d6e5 100644 --- a/ScoutSuite/providers/azure/resources/keyvault/key_vaults.py +++ b/ScoutSuite/providers/azure/resources/keyvault/key_vaults.py @@ -4,19 +4,18 @@ class KeyVaults(Resources): - async def fetch_all(self, credentials, **kwargs): # TODO: build that facade somewhere else: facade = KeyVaultFacade(credentials.credentials, credentials.subscription_id) self['vaults'] = {} for raw_vault in await facade.get_key_vaults(): - id, vault = self._parse(raw_vault) + id, vault = self._parse_key_vault(raw_vault) self['vaults'][id] = vault self['vaults_count'] = len(self['vaults']) - def _parse(self, raw_vault): + def _parse_key_vault(self, raw_vault): vault = {} vault['id'] = get_non_provider_id(raw_vault.id) vault['name'] = raw_vault.name From b725d95ab98099c6c8998f2ab320f4266be89364 Mon Sep 17 00:00:00 2001 From: misg Date: Wed, 13 Mar 2019 13:31:54 -0400 Subject: [PATCH 369/667] Rename _parse method. --- .../providers/azure/resources/monitor/activity_logs.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/monitor/activity_logs.py b/ScoutSuite/providers/azure/resources/monitor/activity_logs.py index a48390b36..4ad2ad444 100644 --- a/ScoutSuite/providers/azure/resources/monitor/activity_logs.py +++ b/ScoutSuite/providers/azure/resources/monitor/activity_logs.py @@ -4,7 +4,6 @@ class ActivityLogs(Resources): - async def fetch_all(self, credentials, **kwargs): # TODO: build that facade somewhere else: facade = MonitorFacade(credentials.credentials, credentials.subscription_id) @@ -13,9 +12,9 @@ async def fetch_all(self, credentials, **kwargs): self['activity_logs']['storage_accounts'] = {} for raw_log in await facade.get_activity_logs(): - self._parse(raw_log) + self._parse_activity_log(raw_log) - def _parse(self, raw_log): + def _parse_activity_log(self, raw_log): if raw_log.resource_type.value == 'Microsoft.Storage/storageAccounts': self._parse_storage_account_log(raw_log) From 4176f9374b42f6410a8d6bf5d680fc137448d7dd Mon Sep 17 00:00:00 2001 From: misg Date: Wed, 13 Mar 2019 13:40:36 -0400 Subject: [PATCH 370/667] Rename _parse methods. --- .../azure/resources/storageaccounts/blob_containers.py | 5 ++--- .../azure/resources/storageaccounts/storageaccounts.py | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/storageaccounts/blob_containers.py b/ScoutSuite/providers/azure/resources/storageaccounts/blob_containers.py index 8716c463b..e8e182dbb 100644 --- a/ScoutSuite/providers/azure/resources/storageaccounts/blob_containers.py +++ b/ScoutSuite/providers/azure/resources/storageaccounts/blob_containers.py @@ -2,7 +2,6 @@ class BlobContainers(Resources): - def __init__(self, resource_group_name, storage_account_name, facade): self.resource_group_name = resource_group_name self.storage_account_name = storage_account_name @@ -13,10 +12,10 @@ async def fetch_all(self): self.resource_group_name, self.storage_account_name ) for raw_blob_container in raw_blob_containers: - id, blob_container = self._parse(raw_blob_container) + id, blob_container = self._parse_blob_container(raw_blob_container) self[id] = blob_container - def _parse(self, raw_blob_container): + def _parse_blob_container(self, raw_blob_container): blob_container = {} blob_container['id'] = raw_blob_container.name blob_container['public_access_allowed'] = raw_blob_container.public_access != "None" diff --git a/ScoutSuite/providers/azure/resources/storageaccounts/storageaccounts.py b/ScoutSuite/providers/azure/resources/storageaccounts/storageaccounts.py index 6ccd6134b..f96f5f80d 100644 --- a/ScoutSuite/providers/azure/resources/storageaccounts/storageaccounts.py +++ b/ScoutSuite/providers/azure/resources/storageaccounts/storageaccounts.py @@ -19,7 +19,7 @@ async def fetch_all(self, credentials, **kwargs): self['storage_accounts'] = {} for raw_storage_account in await facade.get_storage_accounts(): - id, storage_account = self._parse(raw_storage_account) + id, storage_account = self._parse_storage_account(raw_storage_account) self['storage_accounts'][id] = storage_account # TODO: make a refactoring of the following: @@ -39,7 +39,7 @@ async def fetch_all(self, credentials, **kwargs): self['storage_accounts_count'] = len(self['storage_accounts']) - def _parse(self, raw_storage_account): + def _parse_storage_account(self, raw_storage_account): storage_account = {} raw_id = raw_storage_account.id storage_account['id'] = get_non_provider_id(raw_id.lower()) From 6a3ebef7fb1611b974a8f8e3814e0b167d6ed1d8 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 13 Mar 2019 16:28:13 -0400 Subject: [PATCH 371/667] Started implementing IAM facade --- ScoutSuite/providers/aws/facade/iam.py | 130 +++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 ScoutSuite/providers/aws/facade/iam.py diff --git a/ScoutSuite/providers/aws/facade/iam.py b/ScoutSuite/providers/aws/facade/iam.py new file mode 100644 index 000000000..91643264f --- /dev/null +++ b/ScoutSuite/providers/aws/facade/iam.py @@ -0,0 +1,130 @@ +from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils +from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade +from ScoutSuite.core.console import print_error, print_exception +from ScoutSuite.providers.utils import run_concurrently, get_non_provider_id +from ScoutSuite.providers.aws.utils import is_throttled + + +class IAMFacade(AWSBaseFacade): + async def get_credential_reports(self): + client = AWSFacadeUtils.get_client('iam', self.session) + response = client.generate_credential_report() + + if response['State'] != 'COMPLETE': + print_error('Failed to generate a credential report.') + return [] + + report = client.get_credential_report()['Content'] + + # The report is a CSV string. The first row contains the name of each column. The next rows + # each represent an individual account. This algorithm provides a simple initial parsing. + lines = report.splitlines() + keys = lines[0].decode('utf-8').split(',') + + credential_reports = [] + for line in lines[1:]: + credential_report = {} + values = line.decode('utf-8').split(',') + for key, value in zip(keys, values): + credential_report[key] = value + + credential_reports.append(credential_report) + + return credential_reports + + async def get_groups(self): + groups = await AWSFacadeUtils.get_all_pages('iam', None, self.session, 'list_groups', 'Groups') + for group in groups: + group['Users'] = await self._fetch_group_users(group['GroupName']) + policies = self._get_inline_policies('group', group['GroupId'], group['GroupName']) + if len(policies): + group['inline_policies'] = policies + return groups + + async def _fetch_group_users(self, group_name): + client = AWSFacadeUtils.get_client('iam', self.session) + fetched_users = client.get_group(GroupName=group_name)['Users'] + + users = [] + for user in fetched_users: + users.append(user['UserId']) + return users + + def _get_inline_policies(self, iam_resource_type, resource_id, resource_name): + client = AWSFacadeUtils.get_client('iam', self.session) + get_policy_method = getattr(client, 'get_' + iam_resource_type + '_policy') + fetched_policies = {} + list_policy_method = getattr(client, 'list_' + iam_resource_type + '_policies') + args = {iam_resource_type.title() + 'Name': resource_name} + try: + policy_names = list_policy_method(**args)['PolicyNames'] + except Exception as e: + if is_throttled(e): + raise e + else: + print_exception(e) + return fetched_policies + try: + for policy_name in policy_names: + args['PolicyName'] = policy_name + policy_document = get_policy_method(**args)['PolicyDocument'] + policy_id = get_non_provider_id(policy_name) + fetched_policies[policy_id] = {} + fetched_policies[policy_id]['PolicyDocument'] = policy_document + fetched_policies[policy_id]['name'] = policy_name + # self._parse_permissions(policy_id, policy_document, 'inline_policies', iam_resource_type + 's', resource_id) + except Exception as e: + if is_throttled(e): + raise e + else: + print_exception(e) + return fetched_policies + + # def _parse_permissions(self, policy_name, policy_document, policy_type, iam_resource_type, resource_name): + # # Enforce list of statements (Github issue #99) + # if type(policy_document['Statement']) != list: + # policy_document['Statement'] = [policy_document['Statement']] + # for statement in policy_document['Statement']: + # self._parse_statement(policy_name, statement, policy_type, iam_resource_type, resource_name) + + # def _parse_statement(self, policy_name, statement, policy_type, iam_resource_type, resource_name): + # # Effect + # effect = str(statement['Effect']) + # # Action or NotAction + # action_string = 'Action' if 'Action' in statement else 'NotAction' + # if type(statement[action_string]) != list: + # statement[action_string] = [statement[action_string]] + # # Resource or NotResource + # resource_string = 'Resource' if 'Resource' in statement else 'NotResource' + # if type(statement[resource_string]) != list: + # statement[resource_string] = [statement[resource_string]] + # # Condition + # condition = statement['Condition'] if 'Condition' in statement else None + # if iam_resource_type is None: + # return + + # self.__parse_actions(effect, action_string, statement[action_string], resource_string, + # statement[resource_string], iam_resource_type, resource_name, policy_name, policy_type, + # condition) + + # def _parse_action(self, effect, action_string, action, resource_string, resources, iam_resource_type, + # iam_resource_name, policy_name, policy_type, condition): + # for resource in resources: + # self.__parse_resource(effect, action_string, action, resource_string, resource, iam_resource_type, + # iam_resource_name, policy_name, policy_type, condition) + + # def _parse_resource(self, effect, action_string, action, resource_string, resource, iam_resource_type, + # iam_resource_name, policy_name, policy_type, condition): + # manage_dictionary(self.permissions[action_string][action][iam_resource_type][effect][iam_resource_name], + # resource_string, {}) + # manage_dictionary( + # self.permissions[action_string][action][iam_resource_type][effect][iam_resource_name][resource_string], + # resource, {}) + # manage_dictionary( + # self.permissions[action_string][action][iam_resource_type][effect][iam_resource_name][resource_string][ + # resource], policy_type, {}) + # manage_dictionary( + # self.permissions[action_string][action][iam_resource_type][effect][iam_resource_name][resource_string][ + # resource][policy_type], policy_name, {}) + # self.permissions[action_string][action][iam_resource_type][effect][iam_resource_name][resource_string][ + # resource][policy_type][policy_name]['condition'] = condition \ No newline at end of file From a7e38b7014dc323991ac0f646a6f1f95e451ad1a Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 13 Mar 2019 16:28:24 -0400 Subject: [PATCH 372/667] Integrated IAM facade to AWS facade --- ScoutSuite/providers/aws/facade/facade.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ScoutSuite/providers/aws/facade/facade.py b/ScoutSuite/providers/aws/facade/facade.py index e0feccb46..66e7be9b8 100644 --- a/ScoutSuite/providers/aws/facade/facade.py +++ b/ScoutSuite/providers/aws/facade/facade.py @@ -10,6 +10,7 @@ from ScoutSuite.providers.aws.facade.ec2 import EC2Facade from ScoutSuite.providers.aws.facade.efs import EFSFacade from ScoutSuite.providers.aws.facade.elasticache import ElastiCacheFacade +from ScoutSuite.providers.aws.facade.iam import IAMFacade from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade from ScoutSuite.providers.utils import run_concurrently from ScoutSuite.core.console import print_error, print_debug @@ -27,6 +28,7 @@ def __init__(self, credentials: dict = None): self.directconnect = DirectConnectFacade(self.session) self.efs = EFSFacade(self.session) self.elasticache = ElastiCacheFacade(self.session) + self.iam = IAMFacade(self.session) async def build_region_list(self, service: str, chosen_regions=None, partition_name='aws'): service = 'ec2containerservice' if service == 'ecs' else service @@ -64,3 +66,4 @@ def _set_session(self, credentials: dict): self.directconnect = DirectConnectFacade(self.session) self.efs = EFSFacade(self.session) self.elasticache = ElastiCacheFacade(self.session) + self.iam = IAMFacade(self.session) From 3c2bc820e785da227932c2bde5132312200332ed Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 13 Mar 2019 16:28:32 -0400 Subject: [PATCH 373/667] Created credential reports node --- .../aws/resources/iam/credentialreports.py | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 ScoutSuite/providers/aws/resources/iam/credentialreports.py diff --git a/ScoutSuite/providers/aws/resources/iam/credentialreports.py b/ScoutSuite/providers/aws/resources/iam/credentialreports.py new file mode 100644 index 000000000..683a6d5c9 --- /dev/null +++ b/ScoutSuite/providers/aws/resources/iam/credentialreports.py @@ -0,0 +1,35 @@ +from ScoutSuite.providers.aws.resources.resources import AWSResources + + +class CredentialReports(AWSResources): + async def fetch_all(self, **kwargs): + raw_credential_reports = await self.facade.iam.get_credential_reports() + for raw_credential_report in raw_credential_reports: + name, resource = self._parse_credential_reports(raw_credential_report) + self[name] = resource + + def _parse_credential_reports(self, raw_credential_report): + user_id = raw_credential_report['user'] + raw_credential_report['name'] = user_id + raw_credential_report['id'] = user_id + raw_credential_report['password_last_used'] = self._sanitize_date(raw_credential_report['password_last_used']) + raw_credential_report['access_key_1_last_used_date'] = self._sanitize_date(raw_credential_report['access_key_1_last_used_date']) + raw_credential_report['access_key_2_last_used_date'] = self._sanitize_date(raw_credential_report['access_key_2_last_used_date']) + raw_credential_report['last_used'] = self._compute_last_used(raw_credential_report) + return user_id, raw_credential_report + + @staticmethod + def _sanitize_date(date): + """ + Returns the date if it is not equal to 'N/A' or 'no_information', else returns None + """ + return date if date != 'no_information' and date != 'N/A' else None + + @staticmethod + def _compute_last_used(credential_report): + dates = [credential_report['password_last_used'], + credential_report['access_key_1_last_used_date'], + credential_report['access_key_2_last_used_date']] + + dates = [date for date in dates if date is not None] + return max(dates) if len(dates) > 0 else None \ No newline at end of file From 3a8eea73ddbcb33eed6248021c1f31958a86e19a Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 13 Mar 2019 16:29:04 -0400 Subject: [PATCH 374/667] Started implementing group node --- .../providers/aws/resources/iam/groups.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 ScoutSuite/providers/aws/resources/iam/groups.py diff --git a/ScoutSuite/providers/aws/resources/iam/groups.py b/ScoutSuite/providers/aws/resources/iam/groups.py new file mode 100644 index 000000000..6c71d4602 --- /dev/null +++ b/ScoutSuite/providers/aws/resources/iam/groups.py @@ -0,0 +1,19 @@ +from ScoutSuite.providers.aws.resources.resources import AWSResources + + +class Groups(AWSResources): + async def fetch_all(self, **kwargs): + raw_groups = await self.facade.iam.get_groups() + for raw_group in raw_groups: + name, resource = self._parse_group(raw_group) + self[name] = resource + + def _parse_group(self, raw_group): + if raw_group['GroupName'] in self: + return + + raw_group['id'] = raw_group.pop('GroupId') + raw_group['name'] = raw_group.pop('GroupName') + raw_group['arn'] = raw_group.pop('Arn') + raw_group['users'] = raw_group.pop('Users') + return raw_group['id'], raw_group From ec21eb1d0d376ab055ff425e2cb87469950222b5 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 13 Mar 2019 16:29:35 -0400 Subject: [PATCH 375/667] Allowed empty region in get_client --- ScoutSuite/providers/aws/facade/awslambda.py | 1 + ScoutSuite/providers/aws/facade/cloudformation.py | 2 +- ScoutSuite/providers/aws/facade/cloudtrail.py | 2 +- ScoutSuite/providers/aws/facade/directconnect.py | 2 +- ScoutSuite/providers/aws/facade/ec2.py | 6 +++--- ScoutSuite/providers/aws/facade/efs.py | 2 +- ScoutSuite/providers/aws/facade/elasticache.py | 2 +- ScoutSuite/providers/aws/facade/utils.py | 6 +++--- 8 files changed, 12 insertions(+), 11 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/awslambda.py b/ScoutSuite/providers/aws/facade/awslambda.py index 893b67cbd..3a7662255 100644 --- a/ScoutSuite/providers/aws/facade/awslambda.py +++ b/ScoutSuite/providers/aws/facade/awslambda.py @@ -1,6 +1,7 @@ from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade + class LambdaFacade(AWSBaseFacade): async def get_functions(self, region): return await AWSFacadeUtils.get_all_pages('lambda', region, self.session, 'list_functions', 'Functions') diff --git a/ScoutSuite/providers/aws/facade/cloudformation.py b/ScoutSuite/providers/aws/facade/cloudformation.py index 99408ca3d..db7e04f64 100644 --- a/ScoutSuite/providers/aws/facade/cloudformation.py +++ b/ScoutSuite/providers/aws/facade/cloudformation.py @@ -9,7 +9,7 @@ class CloudFormation(AWSBaseFacade): async def get_stacks(self, region: str): stacks = await AWSFacadeUtils.get_all_pages('cloudformation', region, self.session, 'list_stacks', 'StackSummaries') stacks = [stack for stack in stacks if not CloudFormation._is_stack_deleted(stack)] - client = AWSFacadeUtils.get_client('cloudformation', region, self.session) + client = AWSFacadeUtils.get_client('cloudformation', self.session, region) for stack in stacks: stack_name = stack['StackName'] diff --git a/ScoutSuite/providers/aws/facade/cloudtrail.py b/ScoutSuite/providers/aws/facade/cloudtrail.py index f34392ba5..78ba4cbc3 100644 --- a/ScoutSuite/providers/aws/facade/cloudtrail.py +++ b/ScoutSuite/providers/aws/facade/cloudtrail.py @@ -5,7 +5,7 @@ class CloudTrailFacade(AWSBaseFacade): async def get_trails(self, region): - client = AWSFacadeUtils.get_client('cloudtrail', region, self.session) + client = AWSFacadeUtils.get_client('cloudtrail', self.session, region) trails = await run_concurrently( lambda: client.describe_trails()['trailList'] ) diff --git a/ScoutSuite/providers/aws/facade/directconnect.py b/ScoutSuite/providers/aws/facade/directconnect.py index df24839cb..a4ca2fb61 100644 --- a/ScoutSuite/providers/aws/facade/directconnect.py +++ b/ScoutSuite/providers/aws/facade/directconnect.py @@ -5,5 +5,5 @@ class DirectConnectFacade(AWSBaseFacade): async def get_connections(self, region): - client = AWSFacadeUtils.get_client('directconnect', region, self.session) + client = AWSFacadeUtils.get_client('directconnect', self.session, region) return await run_concurrently(lambda: client.describe_connections()['connections']) diff --git a/ScoutSuite/providers/aws/facade/ec2.py b/ScoutSuite/providers/aws/facade/ec2.py index 704b61358..673413396 100644 --- a/ScoutSuite/providers/aws/facade/ec2.py +++ b/ScoutSuite/providers/aws/facade/ec2.py @@ -8,7 +8,7 @@ class EC2Facade(AWSBaseFacade): async def get_instance_user_data(self, region: str, instance_id: str): - ec2_client = AWSFacadeUtils.get_client('ec2', region, self.session) + ec2_client = AWSFacadeUtils.get_client('ec2', self.session, region) user_data_response = await run_concurrently( lambda: ec2_client.describe_instance_attribute(Attribute='userData', InstanceId=instance_id)) @@ -41,7 +41,7 @@ async def get_vpcs(self, region): async def get_images(self, region, owner_id): filters = [{'Name': 'owner-id', 'Values': [owner_id]}] - client = AWSFacadeUtils.get_client('ec2', region, self.session) + client = AWSFacadeUtils.get_client('ec2', self.session, region) response = await run_concurrently(lambda: client.describe_images(Filters=filters)) return response['Images'] @@ -59,7 +59,7 @@ async def get_snapshots(self, region, owner_id): snapshots = await AWSFacadeUtils.get_all_pages( 'ec2', region, self.session, 'describe_snapshots', 'Snapshots', Filters=filters) - ec2_client = AWSFacadeUtils.get_client('ec2', region, self.session) + ec2_client = AWSFacadeUtils.get_client('ec2', self.session, region) for snapshot in snapshots: snapshot['CreateVolumePermissions'] = await run_concurrently(lambda: ec2_client.describe_snapshot_attribute( Attribute='createVolumePermission', diff --git a/ScoutSuite/providers/aws/facade/efs.py b/ScoutSuite/providers/aws/facade/efs.py index b33955d89..08268c283 100644 --- a/ScoutSuite/providers/aws/facade/efs.py +++ b/ScoutSuite/providers/aws/facade/efs.py @@ -8,7 +8,7 @@ class EFSFacade(AWSBaseFacade): async def get_file_systems(self, region: str): file_systems = await AWSFacadeUtils.get_all_pages('efs', region, self.session, 'describe_file_systems', 'FileSystems') - client = AWSFacadeUtils.get_client('efs', region, self.session) + client = AWSFacadeUtils.get_client('efs', self.session, region) for file_system in file_systems: file_system_id = file_system['FileSystemId'] file_system['Tags'] = await run_concurrently(lambda: client.describe_tags(FileSystemId=file_system_id)['Tags']) diff --git a/ScoutSuite/providers/aws/facade/elasticache.py b/ScoutSuite/providers/aws/facade/elasticache.py index 6f3e07a9e..d7a6bb2fb 100644 --- a/ScoutSuite/providers/aws/facade/elasticache.py +++ b/ScoutSuite/providers/aws/facade/elasticache.py @@ -30,7 +30,7 @@ async def cache_clusters(self, region): cluster['VpcId'] = subnet_group['VpcId'] async def get_security_groups(self, region): - client = AWSFacadeUtils.get_client('elasticache', region, self.session) + client = AWSFacadeUtils.get_client('elasticache', self.session, region) try: return await AWSFacadeUtils.get_all_pages('elasticache', region, self.session, 'describe_cache_security_groups', 'CacheSecurityGroups') diff --git a/ScoutSuite/providers/aws/facade/utils.py b/ScoutSuite/providers/aws/facade/utils.py index 94c0e189b..13bc0fec8 100644 --- a/ScoutSuite/providers/aws/facade/utils.py +++ b/ScoutSuite/providers/aws/facade/utils.py @@ -9,7 +9,7 @@ class AWSFacadeUtils: @staticmethod async def get_all_pages(service: str, region: str, session: boto3.session.Session, paginator_name: str, response_key: str, **paginator_args): - client = AWSFacadeUtils.get_client(service, region, session) + client = AWSFacadeUtils.get_client(service, session, region) # Building a paginator doesn't require any API call so no need to do it concurrently: paginator = client.get_paginator(paginator_name).paginate(**paginator_args) @@ -26,7 +26,7 @@ def _get_all_pages_from_paginator(paginator, key): return resources @staticmethod - def get_client(service: str, region: str, session: boto3.session.Session): + def get_client(service: str, session: boto3.session.Session, region: str=None): """ Instantiates an AWS API client @@ -38,5 +38,5 @@ def get_client(service: str, region: str, session: boto3.session.Session): """ # TODO: investigate the use of a mutex to avoid useless creation of a same type of client among threads: - client = session.client(service, region_name=region) + client = session.client(service, region_name=region) if region is not None else session.client(service) return AWSFacadeUtils._clients.setdefault((service, region), client) From ad46e848b62c8b09a5325256b58cc4f08caf8432 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 13 Mar 2019 16:30:12 -0400 Subject: [PATCH 376/667] Started IAM service migration --- ScoutSuite/providers/aws/configs/services.py | 7 +++--- .../providers/aws/resources/iam/service.py | 23 +++++++++++++++++++ 2 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 ScoutSuite/providers/aws/resources/iam/service.py diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index 16c1b1351..a7a057822 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -11,7 +11,7 @@ from ScoutSuite.providers.aws.services.elb import ELBConfig from ScoutSuite.providers.aws.services.elbv2 import ELBv2Config from ScoutSuite.providers.aws.services.emr import EMRConfig -from ScoutSuite.providers.aws.services.iam import IAMConfig +from ScoutSuite.providers.aws.resources.iam.service import IAM from ScoutSuite.providers.aws.services.rds import RDSConfig from ScoutSuite.providers.aws.services.redshift import RedshiftConfig from ScoutSuite.providers.aws.services.route53 import Route53Config, Route53DomainsConfig @@ -64,9 +64,10 @@ def __init__(self, metadata=None, thread_config=4, **kwargs): self.elb = ELBConfig(metadata['compute']['elb'], thread_config) self.elbv2 = ELBv2Config(metadata['compute']['elbv2'], thread_config) self.emr = EMRConfig(metadata['analytics']['emr'], thread_config) - self.iam = IAMConfig(thread_config) + self.iam = IAM() self.awslambda = Lambdas() - self.redshift = RedshiftConfig(metadata['database']['redshift'], thread_config) + self.redshift = RedshiftConfig( + metadata['database']['redshift'], thread_config) self.rds = RDSConfig(metadata['database']['rds'], thread_config) self.route53 = Route53Config(thread_config) self.route53domains = Route53DomainsConfig(thread_config) diff --git a/ScoutSuite/providers/aws/resources/iam/service.py b/ScoutSuite/providers/aws/resources/iam/service.py new file mode 100644 index 000000000..51aad729e --- /dev/null +++ b/ScoutSuite/providers/aws/resources/iam/service.py @@ -0,0 +1,23 @@ +import asyncio + +from ScoutSuite.providers.aws.resources.resources import AWSCompositeResources +from ScoutSuite.providers.aws.resources.iam.credentialreports import CredentialReports +from ScoutSuite.providers.aws.resources.iam.groups import Groups +from ScoutSuite.providers.aws.facade.facade import AWSFacade + + +class IAM(AWSCompositeResources): + _children = [ + (CredentialReports, 'credential_reports'), + (Groups, 'groups') + ] + + def __init__(self): + # TODO: Should be injected + self.facade = AWSFacade() + self.service = 'iam' + + async def fetch_all(self, credentials, regions=None, partition_name='aws'): + # TODO: This should not be set here, the facade should be injected and already authenticated + self.facade._set_session(credentials) + await self._fetch_children(self, {}) From a8d4484ff9cdf344577485061013d549dfa0ec49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sgha=C3=AFer?= Date: Wed, 13 Mar 2019 17:04:55 -0400 Subject: [PATCH 377/667] Made describe_foobar calls async Co-Authored-By: Aboisier --- ScoutSuite/providers/aws/facade/emr.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/emr.py b/ScoutSuite/providers/aws/facade/emr.py index 18280a789..dea818190 100644 --- a/ScoutSuite/providers/aws/facade/emr.py +++ b/ScoutSuite/providers/aws/facade/emr.py @@ -8,8 +8,14 @@ async def get_clusters(self, region): clusters_list = await AWSFacadeUtils.get_all_pages('emr', region, self.session, 'list_clusters', 'Clusters') client = AWSFacadeUtils.get_client('emr', region, self.session) clusters_descriptions = [] - for cluster_id in [cluster['Id'] for cluster in clusters_list]: - cluster = client.describe_cluster(ClusterId=cluster_id)['Cluster'] - clusters_descriptions.append(cluster) + cluster_ids = [cluster['Id'] for cluster in clusters_list] + tasks = { + asyncio.ensure_future( + run_concurrently(lambda: client.describe_cluster(ClusterId=cluster_id)['Cluster']) + ) for cluster_id in cluster_ids + } + for task in asyncio.as_completed(tasks): + cluster = await task + clusters_descriptions.append(cluster) return clusters_descriptions From a5f3dca1a75aa3e7f90ff753aa1c4ac6defeb6ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sgha=C3=AFer?= Date: Wed, 13 Mar 2019 17:05:09 -0400 Subject: [PATCH 378/667] Added missing import Co-Authored-By: Aboisier --- ScoutSuite/providers/aws/facade/emr.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ScoutSuite/providers/aws/facade/emr.py b/ScoutSuite/providers/aws/facade/emr.py index dea818190..b09434ed2 100644 --- a/ScoutSuite/providers/aws/facade/emr.py +++ b/ScoutSuite/providers/aws/facade/emr.py @@ -1,6 +1,7 @@ from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade from ScoutSuite.providers.utils import run_concurrently +import asyncio class EMRFacade(AWSBaseFacade): From 1a0e6a3dfed9c1bc45f5fd69163529de6eabbe7f Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 13 Mar 2019 20:35:30 -0400 Subject: [PATCH 379/667] Updated EMR facade to work with new get_client --- ScoutSuite/providers/aws/facade/emr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/facade/emr.py b/ScoutSuite/providers/aws/facade/emr.py index b09434ed2..ebeb6b448 100644 --- a/ScoutSuite/providers/aws/facade/emr.py +++ b/ScoutSuite/providers/aws/facade/emr.py @@ -7,7 +7,7 @@ class EMRFacade(AWSBaseFacade): async def get_clusters(self, region): clusters_list = await AWSFacadeUtils.get_all_pages('emr', region, self.session, 'list_clusters', 'Clusters') - client = AWSFacadeUtils.get_client('emr', region, self.session) + client = AWSFacadeUtils.get_client('emr', self.session, region) clusters_descriptions = [] cluster_ids = [cluster['Id'] for cluster in clusters_list] tasks = { From 25b940f9e586463388b8bd5b46ac4da7439ddbdb Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 13 Mar 2019 20:35:47 -0400 Subject: [PATCH 380/667] Added method to get multiple entities from all pages --- ScoutSuite/providers/aws/facade/utils.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/utils.py b/ScoutSuite/providers/aws/facade/utils.py index 13bc0fec8..fd3d0aceb 100644 --- a/ScoutSuite/providers/aws/facade/utils.py +++ b/ScoutSuite/providers/aws/facade/utils.py @@ -8,25 +8,34 @@ class AWSFacadeUtils: _clients = {} @staticmethod - async def get_all_pages(service: str, region: str, session: boto3.session.Session, paginator_name: str, response_key: str, **paginator_args): + async def get_all_pages(service: str, region: str, session: boto3.session.Session, paginator_name: str, entity: str, **paginator_args): + results = await AWSFacadeUtils.get_multiple_entities_from_all_pages(service, region, session, paginator_name, [entity], **paginator_args) + return results[entity] + + @staticmethod + async def get_multiple_entities_from_all_pages(service: str, region: str, session: boto3.session.Session, paginator_name: str, entities: list, **paginator_args): client = AWSFacadeUtils.get_client(service, session, region) + # Building a paginator doesn't require any API call so no need to do it concurrently: - paginator = client.get_paginator(paginator_name).paginate(**paginator_args) + paginator = client.get_paginator( + paginator_name).paginate(**paginator_args) # Getting all pages from a paginator requires API calls so we need to do it concurrently: - return await run_concurrently(lambda: AWSFacadeUtils._get_all_pages_from_paginator(paginator, response_key)) + return await run_concurrently(lambda: AWSFacadeUtils._get_all_pages_from_paginator(paginator, entities)) @staticmethod - def _get_all_pages_from_paginator(paginator, key): - resources = [] + def _get_all_pages_from_paginator(paginator, entities: list): + resources = {entity: [] for entity in entities} + # There's an API call hidden behind each iteration: for page in paginator: - resources.extend(page[key]) + for entity in entities: + resources[entity].extend(page[entity]) return resources @staticmethod - def get_client(service: str, session: boto3.session.Session, region: str=None): + def get_client(service: str, session: boto3.session.Session, region: str = None): """ Instantiates an AWS API client From 38ee7036408ffe75116b0f5a50717bcaf89a04e6 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 13 Mar 2019 20:35:59 -0400 Subject: [PATCH 381/667] Added get policies method --- ScoutSuite/providers/aws/facade/iam.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/ScoutSuite/providers/aws/facade/iam.py b/ScoutSuite/providers/aws/facade/iam.py index 91643264f..cc6399718 100644 --- a/ScoutSuite/providers/aws/facade/iam.py +++ b/ScoutSuite/providers/aws/facade/iam.py @@ -41,6 +41,24 @@ async def get_groups(self): group['inline_policies'] = policies return groups + async def get_policies(self): + policies = await AWSFacadeUtils.get_all_pages('iam', None, self.session, 'list_policies', 'Policies', OnlyAttached=True) + for policy in policies: + policy['attached_to'] = {} + attached_entities = await AWSFacadeUtils.get_multiple_entities_from_all_pages('iam', None, self.session, 'list_entities_for_policy', ['PolicyGroups', 'PolicyRoles', 'PolicyUsers'], PolicyArn=policy['Arn']) + + for entity_type in attached_entities: + resource_type = entity_type.replace('Policy', '').lower() + if len(attached_entities[entity_type]): + policy['attached_to'][resource_type] = [] + + for entity in attached_entities[entity_type]: + name_field = entity_type.replace('Policy', '')[:-1] + 'Name' + resource_name = entity[name_field] + policy['attached_to'][resource_type].append({'name': resource_name}) + + return policies + async def _fetch_group_users(self, group_name): client = AWSFacadeUtils.get_client('iam', self.session) fetched_users = client.get_group(GroupName=group_name)['Users'] From d4f24ec484cb038909b19a0f6d0ec70322af5fe9 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 13 Mar 2019 20:36:10 -0400 Subject: [PATCH 382/667] Added policies node --- .../providers/aws/resources/iam/policies.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 ScoutSuite/providers/aws/resources/iam/policies.py diff --git a/ScoutSuite/providers/aws/resources/iam/policies.py b/ScoutSuite/providers/aws/resources/iam/policies.py new file mode 100644 index 000000000..7a906a7ee --- /dev/null +++ b/ScoutSuite/providers/aws/resources/iam/policies.py @@ -0,0 +1,15 @@ +from ScoutSuite.providers.aws.resources.resources import AWSResources + + +class Policies(AWSResources): + async def fetch_all(self, **kwargs): + raw_policies = await self.facade.iam.get_policies() + for raw_policy in raw_policies: + name, resource = self._parse_policy(raw_policy) + self[name] = resource + + def _parse_policy(self, raw_policy): + raw_policy['id'] = raw_policy.pop('PolicyId') + raw_policy['name'] = raw_policy.pop('PolicyName') + raw_policy['arn'] = raw_policy.pop('Arn') + return raw_policy['id'], raw_policy From 82b279e461fa51c21969a1e9eb99411c9dcf3c2c Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 13 Mar 2019 20:36:19 -0400 Subject: [PATCH 383/667] Integrated policies do IAM service --- ScoutSuite/providers/aws/resources/iam/service.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/resources/iam/service.py b/ScoutSuite/providers/aws/resources/iam/service.py index 51aad729e..ccf50758a 100644 --- a/ScoutSuite/providers/aws/resources/iam/service.py +++ b/ScoutSuite/providers/aws/resources/iam/service.py @@ -3,13 +3,15 @@ from ScoutSuite.providers.aws.resources.resources import AWSCompositeResources from ScoutSuite.providers.aws.resources.iam.credentialreports import CredentialReports from ScoutSuite.providers.aws.resources.iam.groups import Groups +from ScoutSuite.providers.aws.resources.iam.policies import Policies from ScoutSuite.providers.aws.facade.facade import AWSFacade class IAM(AWSCompositeResources): _children = [ (CredentialReports, 'credential_reports'), - (Groups, 'groups') + (Groups, 'groups'), + (Policies, 'policies') ] def __init__(self): From 1590c9a9335044f57b6ac2944f38f9a26d14b572 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 13 Mar 2019 21:05:23 -0400 Subject: [PATCH 384/667] Added policy document to policies --- ScoutSuite/providers/aws/facade/iam.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ScoutSuite/providers/aws/facade/iam.py b/ScoutSuite/providers/aws/facade/iam.py index cc6399718..7466fb31a 100644 --- a/ScoutSuite/providers/aws/facade/iam.py +++ b/ScoutSuite/providers/aws/facade/iam.py @@ -43,7 +43,11 @@ async def get_groups(self): async def get_policies(self): policies = await AWSFacadeUtils.get_all_pages('iam', None, self.session, 'list_policies', 'Policies', OnlyAttached=True) + client = AWSFacadeUtils.get_client('iam', self.session) for policy in policies: + policy_version = client.get_policy_version(PolicyArn=policy['Arn'], VersionId=policy['DefaultVersionId']) + policy['PolicyDocument'] = policy_version['PolicyVersion']['Document'] + policy['attached_to'] = {} attached_entities = await AWSFacadeUtils.get_multiple_entities_from_all_pages('iam', None, self.session, 'list_entities_for_policy', ['PolicyGroups', 'PolicyRoles', 'PolicyUsers'], PolicyArn=policy['Arn']) From 101ca7e5d287dace62d2b5264ea2b81971fa92dd Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 13 Mar 2019 21:05:34 -0400 Subject: [PATCH 385/667] Added attached entities ids --- ScoutSuite/providers/aws/facade/iam.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/facade/iam.py b/ScoutSuite/providers/aws/facade/iam.py index 7466fb31a..7acc924cf 100644 --- a/ScoutSuite/providers/aws/facade/iam.py +++ b/ScoutSuite/providers/aws/facade/iam.py @@ -59,7 +59,9 @@ async def get_policies(self): for entity in attached_entities[entity_type]: name_field = entity_type.replace('Policy', '')[:-1] + 'Name' resource_name = entity[name_field] - policy['attached_to'][resource_type].append({'name': resource_name}) + id_field = entity_type.replace('Policy', '')[:-1] + 'Id' + resource_id = entity[id_field] + policy['attached_to'][resource_type].append({'name': resource_name, 'id': resource_id}) return policies From 133c1cd91f5fc4dc174e7270daaa1fd9ababc494 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 13 Mar 2019 21:05:41 -0400 Subject: [PATCH 386/667] Lint --- ScoutSuite/providers/aws/facade/iam.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ScoutSuite/providers/aws/facade/iam.py b/ScoutSuite/providers/aws/facade/iam.py index 7acc924cf..a5ed68a95 100644 --- a/ScoutSuite/providers/aws/facade/iam.py +++ b/ScoutSuite/providers/aws/facade/iam.py @@ -44,6 +44,7 @@ async def get_groups(self): async def get_policies(self): policies = await AWSFacadeUtils.get_all_pages('iam', None, self.session, 'list_policies', 'Policies', OnlyAttached=True) client = AWSFacadeUtils.get_client('iam', self.session) + for policy in policies: policy_version = client.get_policy_version(PolicyArn=policy['Arn'], VersionId=policy['DefaultVersionId']) policy['PolicyDocument'] = policy_version['PolicyVersion']['Document'] From 5defc74684b1133f98a810b86a980abc342caae5 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 13 Mar 2019 21:34:33 -0400 Subject: [PATCH 387/667] Added TODO --- ScoutSuite/providers/aws/facade/iam.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ScoutSuite/providers/aws/facade/iam.py b/ScoutSuite/providers/aws/facade/iam.py index a5ed68a95..b30a5649b 100644 --- a/ScoutSuite/providers/aws/facade/iam.py +++ b/ScoutSuite/providers/aws/facade/iam.py @@ -45,6 +45,7 @@ async def get_policies(self): policies = await AWSFacadeUtils.get_all_pages('iam', None, self.session, 'list_policies', 'Policies', OnlyAttached=True) client = AWSFacadeUtils.get_client('iam', self.session) + # TODO: Parallelize this for policy in policies: policy_version = client.get_policy_version(PolicyArn=policy['Arn'], VersionId=policy['DefaultVersionId']) policy['PolicyDocument'] = policy_version['PolicyVersion']['Document'] From 4cefdac50a2a7ea6a95ff045e40ad6f0c20f56d6 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 13 Mar 2019 21:34:40 -0400 Subject: [PATCH 388/667] Lint --- ScoutSuite/providers/aws/facade/iam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/facade/iam.py b/ScoutSuite/providers/aws/facade/iam.py index b30a5649b..0a2d7a789 100644 --- a/ScoutSuite/providers/aws/facade/iam.py +++ b/ScoutSuite/providers/aws/facade/iam.py @@ -68,7 +68,7 @@ async def get_policies(self): return policies async def _fetch_group_users(self, group_name): - client = AWSFacadeUtils.get_client('iam', self.session) + client = AWSFacadeUtils.get_client('iam', self.session) fetched_users = client.get_group(GroupName=group_name)['Users'] users = [] From a4945e895545e0373d84e95e38866144b7c6fb52 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 13 Mar 2019 21:35:45 -0400 Subject: [PATCH 389/667] Implemented get_users method in facade --- ScoutSuite/providers/aws/facade/iam.py | 27 ++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/ScoutSuite/providers/aws/facade/iam.py b/ScoutSuite/providers/aws/facade/iam.py index 0a2d7a789..9b11333ee 100644 --- a/ScoutSuite/providers/aws/facade/iam.py +++ b/ScoutSuite/providers/aws/facade/iam.py @@ -67,6 +67,33 @@ async def get_policies(self): return policies + async def get_users(self): + client = AWSFacadeUtils.get_client('iam', self.session) + users = await AWSFacadeUtils.get_all_pages('iam', None, self.session, 'list_users', 'Users') + + # TODO: Parallelize this + for user in users: + user_name = user['UserName'] + user_id = user['UserId'] + + policies = self._get_inline_policies('user', user_id, user_name) + if len(policies): + user['inline_policies'] = policies + user['inline_policies_count'] = len(policies) + user['groups'] = [] + groups = await AWSFacadeUtils.get_all_pages('iam', None, self.session, 'list_groups_for_user', 'Groups', UserName=user_name) + for group in groups: + user['groups'].append(group['GroupName']) + try: + user['LoginProfile'] = client.get_login_profile(UserName=user_name)[ + 'LoginProfile'] + except Exception: + pass + user['AccessKeys'] = await self._get_user_acces_keys(user_name) + user['MFADevices'] = await self._get_user_mfa_devices(user_name) + + return users + async def _fetch_group_users(self, group_name): client = AWSFacadeUtils.get_client('iam', self.session) fetched_users = client.get_group(GroupName=group_name)['Users'] From 27f76dcb5ac0dcbcb44dbfc17457348fd82998af Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 13 Mar 2019 21:35:53 -0400 Subject: [PATCH 390/667] Removed commented code --- ScoutSuite/providers/aws/facade/iam.py | 49 -------------------------- 1 file changed, 49 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/iam.py b/ScoutSuite/providers/aws/facade/iam.py index 9b11333ee..e2558bb9d 100644 --- a/ScoutSuite/providers/aws/facade/iam.py +++ b/ScoutSuite/providers/aws/facade/iam.py @@ -132,52 +132,3 @@ def _get_inline_policies(self, iam_resource_type, resource_id, resource_name): else: print_exception(e) return fetched_policies - - # def _parse_permissions(self, policy_name, policy_document, policy_type, iam_resource_type, resource_name): - # # Enforce list of statements (Github issue #99) - # if type(policy_document['Statement']) != list: - # policy_document['Statement'] = [policy_document['Statement']] - # for statement in policy_document['Statement']: - # self._parse_statement(policy_name, statement, policy_type, iam_resource_type, resource_name) - - # def _parse_statement(self, policy_name, statement, policy_type, iam_resource_type, resource_name): - # # Effect - # effect = str(statement['Effect']) - # # Action or NotAction - # action_string = 'Action' if 'Action' in statement else 'NotAction' - # if type(statement[action_string]) != list: - # statement[action_string] = [statement[action_string]] - # # Resource or NotResource - # resource_string = 'Resource' if 'Resource' in statement else 'NotResource' - # if type(statement[resource_string]) != list: - # statement[resource_string] = [statement[resource_string]] - # # Condition - # condition = statement['Condition'] if 'Condition' in statement else None - # if iam_resource_type is None: - # return - - # self.__parse_actions(effect, action_string, statement[action_string], resource_string, - # statement[resource_string], iam_resource_type, resource_name, policy_name, policy_type, - # condition) - - # def _parse_action(self, effect, action_string, action, resource_string, resources, iam_resource_type, - # iam_resource_name, policy_name, policy_type, condition): - # for resource in resources: - # self.__parse_resource(effect, action_string, action, resource_string, resource, iam_resource_type, - # iam_resource_name, policy_name, policy_type, condition) - - # def _parse_resource(self, effect, action_string, action, resource_string, resource, iam_resource_type, - # iam_resource_name, policy_name, policy_type, condition): - # manage_dictionary(self.permissions[action_string][action][iam_resource_type][effect][iam_resource_name], - # resource_string, {}) - # manage_dictionary( - # self.permissions[action_string][action][iam_resource_type][effect][iam_resource_name][resource_string], - # resource, {}) - # manage_dictionary( - # self.permissions[action_string][action][iam_resource_type][effect][iam_resource_name][resource_string][ - # resource], policy_type, {}) - # manage_dictionary( - # self.permissions[action_string][action][iam_resource_type][effect][iam_resource_name][resource_string][ - # resource][policy_type], policy_name, {}) - # self.permissions[action_string][action][iam_resource_type][effect][iam_resource_name][resource_string][ - # resource][policy_type][policy_name]['condition'] = condition \ No newline at end of file From ba77f5d4c0f9af0d4f30f523d65cea4bd25bd146 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 13 Mar 2019 21:36:34 -0400 Subject: [PATCH 391/667] Implemented users node --- .../providers/aws/resources/iam/users.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 ScoutSuite/providers/aws/resources/iam/users.py diff --git a/ScoutSuite/providers/aws/resources/iam/users.py b/ScoutSuite/providers/aws/resources/iam/users.py new file mode 100644 index 000000000..2951945e2 --- /dev/null +++ b/ScoutSuite/providers/aws/resources/iam/users.py @@ -0,0 +1,20 @@ +from ScoutSuite.providers.aws.resources.resources import AWSResources + + +class Users(AWSResources): + async def fetch_all(self, **kwargs): + raw_users = await self.facade.iam.get_users() + for raw_user in raw_users: + name, resource = self._parse_user(raw_user) + + if name in self: + continue + + self[name] = resource + + def _parse_user(self, raw_user): + raw_user['id'] = raw_user.pop('UserId') + raw_user['name'] = raw_user.pop('UserName') + raw_user['arn'] = raw_user.pop('Arn') + + return raw_user['id'], raw_user From 0ac861d8481114ebf44e0833d77203499e6d99db Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 13 Mar 2019 21:36:39 -0400 Subject: [PATCH 392/667] Integrated users node --- ScoutSuite/providers/aws/resources/iam/service.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/resources/iam/service.py b/ScoutSuite/providers/aws/resources/iam/service.py index ccf50758a..312d5c0fe 100644 --- a/ScoutSuite/providers/aws/resources/iam/service.py +++ b/ScoutSuite/providers/aws/resources/iam/service.py @@ -4,6 +4,7 @@ from ScoutSuite.providers.aws.resources.iam.credentialreports import CredentialReports from ScoutSuite.providers.aws.resources.iam.groups import Groups from ScoutSuite.providers.aws.resources.iam.policies import Policies +from ScoutSuite.providers.aws.resources.iam.users import Users from ScoutSuite.providers.aws.facade.facade import AWSFacade @@ -11,7 +12,8 @@ class IAM(AWSCompositeResources): _children = [ (CredentialReports, 'credential_reports'), (Groups, 'groups'), - (Policies, 'policies') + (Policies, 'policies'), + (Users, 'users') ] def __init__(self): From 9dea53990f96f76a90e26e1e0a1d9c30c0a72b63 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 13 Mar 2019 21:37:46 -0400 Subject: [PATCH 393/667] Revived missing methods --- ScoutSuite/providers/aws/facade/iam.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ScoutSuite/providers/aws/facade/iam.py b/ScoutSuite/providers/aws/facade/iam.py index e2558bb9d..cf44b7b98 100644 --- a/ScoutSuite/providers/aws/facade/iam.py +++ b/ScoutSuite/providers/aws/facade/iam.py @@ -94,6 +94,17 @@ async def get_users(self): return users + async def _get_user_acces_keys(self, user_name): + client = AWSFacadeUtils.get_client('iam', self.session) + response = await run_concurrently(lambda: client.list_access_keys(UserName=user_name)) + return response['AccessKeyMetadata'] + + async def _get_user_mfa_devices(self, user_name): + client = AWSFacadeUtils.get_client('iam', self.session) + response = await run_concurrently(lambda: client.list_mfa_devices(UserName=user_name)) + return response['MFADevices'] + + async def _fetch_group_users(self, group_name): client = AWSFacadeUtils.get_client('iam', self.session) fetched_users = client.get_group(GroupName=group_name)['Users'] From 29bacdb6692811affda3dfb5bbabe9245f835ade Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 13 Mar 2019 21:46:46 -0400 Subject: [PATCH 394/667] Added TODO --- ScoutSuite/providers/aws/facade/iam.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ScoutSuite/providers/aws/facade/iam.py b/ScoutSuite/providers/aws/facade/iam.py index cf44b7b98..7cfff7d1c 100644 --- a/ScoutSuite/providers/aws/facade/iam.py +++ b/ScoutSuite/providers/aws/facade/iam.py @@ -114,6 +114,7 @@ async def _fetch_group_users(self, group_name): users.append(user['UserId']) return users + # TODO: Make this async def _get_inline_policies(self, iam_resource_type, resource_id, resource_name): client = AWSFacadeUtils.get_client('iam', self.session) get_policy_method = getattr(client, 'get_' + iam_resource_type + '_policy') From c5f4f4472d33ec873c8f5171565e009979d5102d Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 13 Mar 2019 21:47:30 -0400 Subject: [PATCH 395/667] Commented line. Pushing because I'm tired of stashing when I want to switch branches. --- ScoutSuite/providers/aws/provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index cbeae3005..32cbe02a1 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -76,7 +76,7 @@ def preprocessing(self, ip_ranges=None, ip_ranges_name_key=None): self._add_cidr_display_name(ip_ranges, ip_ranges_name_key) self._merge_route53_and_route53domains() - self._match_iam_policies_and_buckets() + # self._match_iam_policies_and_buckets() # TODO: Uncomment this! super(AWSProvider, self).preprocessing() From 5beaa6d405040c92936eb509480821d024b4a57d Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 13 Mar 2019 22:01:01 -0400 Subject: [PATCH 396/667] Added permissions assignation --- .../providers/aws/resources/iam/service.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/ScoutSuite/providers/aws/resources/iam/service.py b/ScoutSuite/providers/aws/resources/iam/service.py index 312d5c0fe..ab738e7a2 100644 --- a/ScoutSuite/providers/aws/resources/iam/service.py +++ b/ScoutSuite/providers/aws/resources/iam/service.py @@ -25,3 +25,26 @@ async def fetch_all(self, credentials, regions=None, partition_name='aws'): # TODO: This should not be set here, the facade should be injected and already authenticated self.facade._set_session(credentials) await self._fetch_children(self, {}) + + def finalize(self): + # Update permissions for managed policies + for policy in self['policies'].values(): + if 'attached_to' in policy and len(policy['attached_to']) > 0: + for entity_type in policy['attached_to']: + for entity in policy['attached_to'][entity_type]: + entity['id'] = self.get_id_for_resource(entity_type, entity['name']) + entities = self[entity_type] + entities[entity['id']]['policies'] = [] # TODO : if does not exist + entities[entity['id']]['policies_counts'] = 0 # TODO : if does not exist + entities[entity['id']]['policies'].append(policy['id']) + entities[entity['id']]['policies_counts'] += 1 + # self.__parse_permissions(policy_id, policy['PolicyDocument'], 'policies', + # entity_type, entity['id']) + else: + pass + # self.__parse_permissions(policy_id, self.policies[policy_id]['PolicyDocument'], 'policies', None, None) + + def get_id_for_resource(self, iam_resource_type, resource_name): + for resource_id in self[iam_resource_type]: + if self[iam_resource_type][resource_id]['name'] == resource_name: + return resource_id From 857ddc1d3cab8a9c585e77be13bf2b968dbd58ea Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 13 Mar 2019 22:31:14 -0400 Subject: [PATCH 397/667] Implemented get_role method --- ScoutSuite/providers/aws/facade/iam.py | 28 ++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/ScoutSuite/providers/aws/facade/iam.py b/ScoutSuite/providers/aws/facade/iam.py index 7cfff7d1c..d058d2a15 100644 --- a/ScoutSuite/providers/aws/facade/iam.py +++ b/ScoutSuite/providers/aws/facade/iam.py @@ -94,6 +94,34 @@ async def get_users(self): return users + async def get_roles(self): + roles = await AWSFacadeUtils.get_all_pages('iam', None, self.session, 'list_roles', 'Roles') + + # Handle throttling errors + for role in roles: + role['instances_count'] = 'N/A' + + # Get role policies + policies = self._get_inline_policies('role', role['RoleId'], role['RoleName']) + if len(policies): + role['inline_policies'] = policies + role['inline_policies_count'] = len(policies) + + # Get instance profiles + profiles = await AWSFacadeUtils.get_all_pages('iam', None, self.session, 'list_instance_profiles_for_role', 'InstanceProfiles', RoleName = role['RoleName']) + role['instance_profiles'] = {} # TODO: If does not exist + for profile in profiles: + profile_id = profile['InstanceProfileId'] + role['instance_profiles'][profile_id] = {} # TODO: If does not exist + role['instance_profiles'][profile_id]['arn'] = profile['Arn'] # TODO: If does not exist + role['instance_profiles'][profile_id]['name'] = profile['InstanceProfileName'] # TODO: If does not exist + + # Get trust relationship + role['assume_role_policy'] = {} + role['assume_role_policy']['PolicyDocument'] = role.pop('AssumeRolePolicyDocument') + + return roles + async def _get_user_acces_keys(self, user_name): client = AWSFacadeUtils.get_client('iam', self.session) response = await run_concurrently(lambda: client.list_access_keys(UserName=user_name)) From 9eda4beb608de9064ab43e2efa2feaf3010fabf6 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 13 Mar 2019 22:31:18 -0400 Subject: [PATCH 398/667] Implemented role node --- ScoutSuite/providers/aws/resources/iam/roles.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 ScoutSuite/providers/aws/resources/iam/roles.py diff --git a/ScoutSuite/providers/aws/resources/iam/roles.py b/ScoutSuite/providers/aws/resources/iam/roles.py new file mode 100644 index 000000000..d4adcf19e --- /dev/null +++ b/ScoutSuite/providers/aws/resources/iam/roles.py @@ -0,0 +1,15 @@ +from ScoutSuite.providers.aws.resources.resources import AWSResources + + +class Roles(AWSResources): + async def fetch_all(self, **kwargs): + raw_roles = await self.facade.iam.get_roles() + for raw_role in raw_roles: + name, resource = self._parse_role(raw_role) + self[name] = resource + + def _parse_role(self, raw_role): + raw_role['id'] = raw_role.pop('RoleId') + raw_role['name'] = raw_role.pop('RoleName') + raw_role['arn'] = raw_role.pop('Arn') + return raw_role['id'], raw_role From 72707f77a1a399a668ad6d431e487e08464bd0e3 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 13 Mar 2019 22:31:29 -0400 Subject: [PATCH 399/667] Integrated role node --- ScoutSuite/providers/aws/resources/iam/service.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/resources/iam/service.py b/ScoutSuite/providers/aws/resources/iam/service.py index ab738e7a2..ce4e8f4c9 100644 --- a/ScoutSuite/providers/aws/resources/iam/service.py +++ b/ScoutSuite/providers/aws/resources/iam/service.py @@ -13,7 +13,8 @@ class IAM(AWSCompositeResources): (CredentialReports, 'credential_reports'), (Groups, 'groups'), (Policies, 'policies'), - (Users, 'users') + (Users, 'users'), + (Roles, 'roles') ] def __init__(self): From da097beea3e0d061bafcdb4e0119d98291d60297 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 13 Mar 2019 22:31:42 -0400 Subject: [PATCH 400/667] Improved finalize to cover more cases --- .../providers/aws/resources/iam/service.py | 63 +++++++++++++++++-- 1 file changed, 57 insertions(+), 6 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/iam/service.py b/ScoutSuite/providers/aws/resources/iam/service.py index ce4e8f4c9..a85621acd 100644 --- a/ScoutSuite/providers/aws/resources/iam/service.py +++ b/ScoutSuite/providers/aws/resources/iam/service.py @@ -5,6 +5,7 @@ from ScoutSuite.providers.aws.resources.iam.groups import Groups from ScoutSuite.providers.aws.resources.iam.policies import Policies from ScoutSuite.providers.aws.resources.iam.users import Users +from ScoutSuite.providers.aws.resources.iam.roles import Roles from ScoutSuite.providers.aws.facade.facade import AWSFacade @@ -30,22 +31,72 @@ async def fetch_all(self, credentials, regions=None, partition_name='aws'): def finalize(self): # Update permissions for managed policies for policy in self['policies'].values(): + policy_id = policy['id'] if 'attached_to' in policy and len(policy['attached_to']) > 0: for entity_type in policy['attached_to']: for entity in policy['attached_to'][entity_type]: - entity['id'] = self.get_id_for_resource(entity_type, entity['name']) + entity['id'] = self._get_id_for_resource(entity_type, entity['name']) entities = self[entity_type] entities[entity['id']]['policies'] = [] # TODO : if does not exist entities[entity['id']]['policies_counts'] = 0 # TODO : if does not exist - entities[entity['id']]['policies'].append(policy['id']) + entities[entity['id']]['policies'].append(policy_id) entities[entity['id']]['policies_counts'] += 1 - # self.__parse_permissions(policy_id, policy['PolicyDocument'], 'policies', - # entity_type, entity['id']) + # self._parse_permissions(policy_id, policy['PolicyDocument'], 'policies', entity_type, entity['id']) else: pass - # self.__parse_permissions(policy_id, self.policies[policy_id]['PolicyDocument'], 'policies', None, None) + # self._parse_permissions(policy_id, policy['PolicyDocument'], 'policies', None, None) - def get_id_for_resource(self, iam_resource_type, resource_name): + def _get_id_for_resource(self, iam_resource_type, resource_name): for resource_id in self[iam_resource_type]: if self[iam_resource_type][resource_id]['name'] == resource_name: return resource_id + + def _parse_permissions(self, policy_name, policy_document, policy_type, iam_resource_type, resource_name): + # Enforce list of statements (Github issue #99) + if type(policy_document['Statement']) != list: + policy_document['Statement'] = [policy_document['Statement']] + for statement in policy_document['Statement']: + self._parse_statement(policy_name, statement, policy_type, iam_resource_type, resource_name) + + def _parse_statement(self, policy_name, statement, policy_type, iam_resource_type, resource_name): + # Effect + effect = str(statement['Effect']) + # Action or NotAction + action_string = 'Action' if 'Action' in statement else 'NotAction' + if type(statement[action_string]) != list: + statement[action_string] = [statement[action_string]] + # Resource or NotResource + resource_string = 'Resource' if 'Resource' in statement else 'NotResource' + if type(statement[resource_string]) != list: + statement[resource_string] = [statement[resource_string]] + # Condition + condition = statement['Condition'] if 'Condition' in statement else None + self['permissions'][action_string] = {} # TODO: If does not exist + if iam_resource_type is None: + return + self._parse_actions(effect, action_string, statement[action_string], resource_string, + statement[resource_string], iam_resource_type, resource_name, policy_name, policy_type, + condition) + + def _parse_actions(self, effect, action_string, actions, resource_string, resources, iam_resource_type, + iam_resource_name, policy_name, policy_type, condition): + for action in actions: + self['permissions'][action_string][action] = {} # TODO: If does not exist + self['permissions'][action_string][action][iam_resource_type] = {} # TODO: If does not exist + self['permissions'][action_string][action][iam_resource_type][effect] = {} # TODO: If does not exist + self['permissions'][action_string][action][iam_resource_type][effect][iam_resource_name] = {} # TODO: If does not exist + self._parse_action(effect, action_string, action, resource_string, resources, iam_resource_type, + iam_resource_name, policy_name, policy_type, condition) + + def _parse_action(self, effect, action_string, action, resource_string, resources, iam_resource_type, + iam_resource_name, policy_name, policy_type, condition): + for resource in resources: + self._parse_resource(effect, action_string, action, resource_string, resource, iam_resource_type, + iam_resource_name, policy_name, policy_type, condition) + + def _parse_resource(self, effect, action_string, action, resource_string, resource, iam_resource_type, iam_resource_name, policy_name, policy_type, condition): + self['permissions'][action_string][action][iam_resource_type][effect][iam_resource_name][resource_string] = {} # TODO: If does not exist + self['permissions'][action_string][action][iam_resource_type][effect][iam_resource_name][resource_string][resource] = {} # TODO: If does not exist + self['permissions'][action_string][action][iam_resource_type][effect][iam_resource_name][resource_string][resource][policy_type] = {} # TODO: If does not exist + self['permissions'][action_string][action][iam_resource_type][effect][iam_resource_name][resource_string][resource][policy_type][policy_name] = {} # TODO: If does not exist + self['permissions'][action_string][action][iam_resource_type][effect][iam_resource_name][resource_string][resource][policy_type][policy_name]['condition'] = condition From a5cf5ed737dbe184ffec796acae3b25c98a8f372 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Thu, 14 Mar 2019 09:07:36 -0400 Subject: [PATCH 401/667] Made sure not to override data --- ScoutSuite/providers/aws/facade/iam.py | 8 +-- .../providers/aws/resources/iam/service.py | 50 ++++++++++--------- 2 files changed, 31 insertions(+), 27 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/iam.py b/ScoutSuite/providers/aws/facade/iam.py index d058d2a15..adbf5edc0 100644 --- a/ScoutSuite/providers/aws/facade/iam.py +++ b/ScoutSuite/providers/aws/facade/iam.py @@ -109,12 +109,12 @@ async def get_roles(self): # Get instance profiles profiles = await AWSFacadeUtils.get_all_pages('iam', None, self.session, 'list_instance_profiles_for_role', 'InstanceProfiles', RoleName = role['RoleName']) - role['instance_profiles'] = {} # TODO: If does not exist + role.setdefault('instance_profiles', {}) for profile in profiles: profile_id = profile['InstanceProfileId'] - role['instance_profiles'][profile_id] = {} # TODO: If does not exist - role['instance_profiles'][profile_id]['arn'] = profile['Arn'] # TODO: If does not exist - role['instance_profiles'][profile_id]['name'] = profile['InstanceProfileName'] # TODO: If does not exist + role['instance_profiles'].setdefault(profile_id, {}) + role['instance_profiles'][profile_id].setdefault('arn', profile['Arn']) + role['instance_profiles'][profile_id].setdefault('name', profile['InstanceProfileName']) # Get trust relationship role['assume_role_policy'] = {} diff --git a/ScoutSuite/providers/aws/resources/iam/service.py b/ScoutSuite/providers/aws/resources/iam/service.py index a85621acd..8736b1a0b 100644 --- a/ScoutSuite/providers/aws/resources/iam/service.py +++ b/ScoutSuite/providers/aws/resources/iam/service.py @@ -30,21 +30,24 @@ async def fetch_all(self, credentials, regions=None, partition_name='aws'): def finalize(self): # Update permissions for managed policies + self['permissions'] = {} for policy in self['policies'].values(): policy_id = policy['id'] if 'attached_to' in policy and len(policy['attached_to']) > 0: for entity_type in policy['attached_to']: for entity in policy['attached_to'][entity_type]: entity['id'] = self._get_id_for_resource(entity_type, entity['name']) - entities = self[entity_type] - entities[entity['id']]['policies'] = [] # TODO : if does not exist - entities[entity['id']]['policies_counts'] = 0 # TODO : if does not exist + entities = self[entity_type] + entities[entity['id']].setdefault('policies', []) + entities[entity['id']].setdefault('policies_counts', 0) entities[entity['id']]['policies'].append(policy_id) entities[entity['id']]['policies_counts'] += 1 - # self._parse_permissions(policy_id, policy['PolicyDocument'], 'policies', entity_type, entity['id']) + self._parse_permissions( + policy_id, policy['PolicyDocument'], 'policies', entity_type, entity['id']) else: pass - # self._parse_permissions(policy_id, policy['PolicyDocument'], 'policies', None, None) + self._parse_permissions( + policy_id, policy['PolicyDocument'], 'policies', None, None) def _get_id_for_resource(self, iam_resource_type, resource_name): for resource_id in self[iam_resource_type]: @@ -56,7 +59,8 @@ def _parse_permissions(self, policy_name, policy_document, policy_type, iam_reso if type(policy_document['Statement']) != list: policy_document['Statement'] = [policy_document['Statement']] for statement in policy_document['Statement']: - self._parse_statement(policy_name, statement, policy_type, iam_resource_type, resource_name) + self._parse_statement(policy_name, statement, + policy_type, iam_resource_type, resource_name) def _parse_statement(self, policy_name, statement, policy_type, iam_resource_type, resource_name): # Effect @@ -71,32 +75,32 @@ def _parse_statement(self, policy_name, statement, policy_type, iam_resource_typ statement[resource_string] = [statement[resource_string]] # Condition condition = statement['Condition'] if 'Condition' in statement else None - self['permissions'][action_string] = {} # TODO: If does not exist + self['permissions'].setdefault(action_string, {}) if iam_resource_type is None: return self._parse_actions(effect, action_string, statement[action_string], resource_string, - statement[resource_string], iam_resource_type, resource_name, policy_name, policy_type, - condition) + statement[resource_string], iam_resource_type, resource_name, policy_name, policy_type, + condition) def _parse_actions(self, effect, action_string, actions, resource_string, resources, iam_resource_type, - iam_resource_name, policy_name, policy_type, condition): + iam_resource_name, policy_name, policy_type, condition): for action in actions: - self['permissions'][action_string][action] = {} # TODO: If does not exist - self['permissions'][action_string][action][iam_resource_type] = {} # TODO: If does not exist - self['permissions'][action_string][action][iam_resource_type][effect] = {} # TODO: If does not exist - self['permissions'][action_string][action][iam_resource_type][effect][iam_resource_name] = {} # TODO: If does not exist - self._parse_action(effect, action_string, action, resource_string, resources, iam_resource_type, - iam_resource_name, policy_name, policy_type, condition) + self['permissions'][action_string].setdefault(action, {}) + self['permissions'][action_string][action].setdefault(iam_resource_type, {}) + self['permissions'][action_string][action][iam_resource_type].setdefault(effect, {}) + self['permissions'][action_string][action][iam_resource_type][effect].setdefault(iam_resource_name, {}) + self._parse_action(effect, action_string, action, resource_string, resources, iam_resource_type, \ + iam_resource_name, policy_name, policy_type, condition) def _parse_action(self, effect, action_string, action, resource_string, resources, iam_resource_type, - iam_resource_name, policy_name, policy_type, condition): + iam_resource_name, policy_name, policy_type, condition): for resource in resources: self._parse_resource(effect, action_string, action, resource_string, resource, iam_resource_type, - iam_resource_name, policy_name, policy_type, condition) + iam_resource_name, policy_name, policy_type, condition) def _parse_resource(self, effect, action_string, action, resource_string, resource, iam_resource_type, iam_resource_name, policy_name, policy_type, condition): - self['permissions'][action_string][action][iam_resource_type][effect][iam_resource_name][resource_string] = {} # TODO: If does not exist - self['permissions'][action_string][action][iam_resource_type][effect][iam_resource_name][resource_string][resource] = {} # TODO: If does not exist - self['permissions'][action_string][action][iam_resource_type][effect][iam_resource_name][resource_string][resource][policy_type] = {} # TODO: If does not exist - self['permissions'][action_string][action][iam_resource_type][effect][iam_resource_name][resource_string][resource][policy_type][policy_name] = {} # TODO: If does not exist - self['permissions'][action_string][action][iam_resource_type][effect][iam_resource_name][resource_string][resource][policy_type][policy_name]['condition'] = condition + self['permissions'][action_string][action][iam_resource_type][effect][iam_resource_name].setdefault(resource_string, {}) + self['permissions'][action_string][action][iam_resource_type][effect][iam_resource_name][resource_string].setdefault(resource, {}) + self['permissions'][action_string][action][iam_resource_type][effect][iam_resource_name][resource_string][resource].setdefault(policy_type, {}) + self['permissions'][action_string][action][iam_resource_type][effect][iam_resource_name][resource_string][resource][policy_type].setdefault(policy_name, {}) + self['permissions'][action_string][action][iam_resource_type][effect][iam_resource_name][resource_string][resource][policy_type][policy_name].setdefault('condition', condition) From 385a7da2f408df53298a0db175d6294edcb84932 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Thu, 14 Mar 2019 10:35:43 -0400 Subject: [PATCH 402/667] Added missing length --- ScoutSuite/providers/aws/facade/iam.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ScoutSuite/providers/aws/facade/iam.py b/ScoutSuite/providers/aws/facade/iam.py index adbf5edc0..a67d15658 100644 --- a/ScoutSuite/providers/aws/facade/iam.py +++ b/ScoutSuite/providers/aws/facade/iam.py @@ -39,6 +39,7 @@ async def get_groups(self): policies = self._get_inline_policies('group', group['GroupId'], group['GroupName']) if len(policies): group['inline_policies'] = policies + group['inline_policies_count'] = len(policies) return groups async def get_policies(self): From e40a5248d1b718601bc33e1429510968002bd2b8 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Thu, 14 Mar 2019 10:39:39 -0400 Subject: [PATCH 403/667] Removed try/catch --- ScoutSuite/providers/aws/provider.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index e4473606a..dcca2c034 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -107,7 +107,6 @@ def _add_security_group_name_to_ec2_grants(self): {'AWSAccountId': self.aws_account_id}) def _add_security_group_data_to_elbv2(self): - none = 'N/A' ec2_config = self.services['ec2'] elbv2_config = self.services['elbv2'] for region in elbv2_config['regions']: @@ -115,13 +114,12 @@ def _add_security_group_data_to_elbv2(self): for lb in elbv2_config['regions'][region]['vpcs'][vpc]['lbs']: for i in range(0, len(elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'])): for sg in ec2_config['regions'][region]['vpcs'][vpc]['security_groups']: - try: - if elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][i][ - 'GroupId'] == sg: - elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][i] = \ - ec2_config['regions'][region]['vpcs'][vpc]['security_groups'][sg] - except KeyError: - pass + if 'GroupId' in elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][i] \ + and elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][i][ + 'GroupId'] == sg: + elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][i] = \ + ec2_config['regions'][region]['vpcs'][vpc]['security_groups'][sg] + check_security_group_rules(elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb], i, 'ingress') check_security_group_rules(elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb], i, 'egress') From 0f43a7c31448e741dadbefce319273a031ead039 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Thu, 14 Mar 2019 11:54:38 -0400 Subject: [PATCH 404/667] Fixed removed id issue --- ScoutSuite/providers/aws/provider.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index dcca2c034..d47b796c8 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -114,11 +114,14 @@ def _add_security_group_data_to_elbv2(self): for lb in elbv2_config['regions'][region]['vpcs'][vpc]['lbs']: for i in range(0, len(elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'])): for sg in ec2_config['regions'][region]['vpcs'][vpc]['security_groups']: - if 'GroupId' in elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][i] \ - and elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][i][ - 'GroupId'] == sg: + group_id = elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][i][ + 'GroupId'] + if 'GroupId' in elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][ + i] and group_id == sg: elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][i] = \ ec2_config['regions'][region]['vpcs'][vpc]['security_groups'][sg] + elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][i][ + 'GroupId'] = group_id check_security_group_rules(elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb], i, 'ingress') check_security_group_rules(elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb], i, 'egress') From 9fad823752077f4bedcd0de909541b8241ab2a00 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Thu, 14 Mar 2019 11:54:50 -0400 Subject: [PATCH 405/667] Removed some attributes --- ScoutSuite/providers/aws/resources/iam/policies.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/iam/policies.py b/ScoutSuite/providers/aws/resources/iam/policies.py index 7a906a7ee..b1a6c6f42 100644 --- a/ScoutSuite/providers/aws/resources/iam/policies.py +++ b/ScoutSuite/providers/aws/resources/iam/policies.py @@ -1,6 +1,5 @@ from ScoutSuite.providers.aws.resources.resources import AWSResources - class Policies(AWSResources): async def fetch_all(self, **kwargs): raw_policies = await self.facade.iam.get_policies() @@ -9,7 +8,11 @@ async def fetch_all(self, **kwargs): self[name] = resource def _parse_policy(self, raw_policy): - raw_policy['id'] = raw_policy.pop('PolicyId') - raw_policy['name'] = raw_policy.pop('PolicyName') - raw_policy['arn'] = raw_policy.pop('Arn') - return raw_policy['id'], raw_policy + policy = {} + policy['id'] = raw_policy.pop('PolicyId') + policy['name'] = raw_policy.pop('PolicyName') + policy['arn'] = raw_policy.pop('Arn') + policy['PolicyDocument'] = raw_policy.pop('PolicyDocument') + policy['attached_to'] = raw_policy.pop('attached_to') + + return policy['id'], policy From 94e7084637af1ecf5d2e21a7972f8e657cbeb714 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Thu, 14 Mar 2019 11:54:56 -0400 Subject: [PATCH 406/667] Removed some attributes --- ScoutSuite/providers/aws/resources/iam/roles.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ScoutSuite/providers/aws/resources/iam/roles.py b/ScoutSuite/providers/aws/resources/iam/roles.py index d4adcf19e..945e5067d 100644 --- a/ScoutSuite/providers/aws/resources/iam/roles.py +++ b/ScoutSuite/providers/aws/resources/iam/roles.py @@ -12,4 +12,6 @@ def _parse_role(self, raw_role): raw_role['id'] = raw_role.pop('RoleId') raw_role['name'] = raw_role.pop('RoleName') raw_role['arn'] = raw_role.pop('Arn') + raw_role.pop('Description') + raw_role.pop('MaxSessionDuration') return raw_role['id'], raw_role From 0165c59c6c859e2acc6a4b2d9175f6bf637bf041 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Thu, 14 Mar 2019 11:55:15 -0400 Subject: [PATCH 407/667] Added inline policies to permissions parsing --- .../providers/aws/resources/iam/service.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/iam/service.py b/ScoutSuite/providers/aws/resources/iam/service.py index 8736b1a0b..502411e01 100644 --- a/ScoutSuite/providers/aws/resources/iam/service.py +++ b/ScoutSuite/providers/aws/resources/iam/service.py @@ -31,7 +31,12 @@ async def fetch_all(self, credentials, regions=None, partition_name='aws'): def finalize(self): # Update permissions for managed policies self['permissions'] = {} - for policy in self['policies'].values(): + policies = [policy for policy in self['policies'].values()] + policies.extend(self._get_inline_policies('groups')) + policies.extend(self._get_inline_policies('users')) + policies.extend(self._get_inline_policies('roles')) + + for policy in policies: policy_id = policy['id'] if 'attached_to' in policy and len(policy['attached_to']) > 0: for entity_type in policy['attached_to']: @@ -42,12 +47,13 @@ def finalize(self): entities[entity['id']].setdefault('policies_counts', 0) entities[entity['id']]['policies'].append(policy_id) entities[entity['id']]['policies_counts'] += 1 - self._parse_permissions( - policy_id, policy['PolicyDocument'], 'policies', entity_type, entity['id']) + self._parse_permissions(policy_id, policy['PolicyDocument'], 'policies', entity_type, entity['id']) else: pass - self._parse_permissions( - policy_id, policy['PolicyDocument'], 'policies', None, None) + self._parse_permissions(policy_id, policy['PolicyDocument'], 'policies', None, None) + + def _get_inline_policies(self, resource_type): + return [resource['inline_policies'] for resource in self[resource_type].values() if 'inline_policies' in resource] def _get_id_for_resource(self, iam_resource_type, resource_name): for resource_id in self[iam_resource_type]: From 61ba8b3ee0d2e7fc74780b752e3f8205bec7e8f2 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Thu, 14 Mar 2019 11:55:23 -0400 Subject: [PATCH 408/667] Removed commented line --- ScoutSuite/providers/aws/facade/iam.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ScoutSuite/providers/aws/facade/iam.py b/ScoutSuite/providers/aws/facade/iam.py index a67d15658..2f84eb9b9 100644 --- a/ScoutSuite/providers/aws/facade/iam.py +++ b/ScoutSuite/providers/aws/facade/iam.py @@ -166,7 +166,6 @@ def _get_inline_policies(self, iam_resource_type, resource_id, resource_name): fetched_policies[policy_id] = {} fetched_policies[policy_id]['PolicyDocument'] = policy_document fetched_policies[policy_id]['name'] = policy_name - # self._parse_permissions(policy_id, policy_document, 'inline_policies', iam_resource_type + 's', resource_id) except Exception as e: if is_throttled(e): raise e From 3ebfe3ae5e640ce945ac1a234326135f71312ecc Mon Sep 17 00:00:00 2001 From: misg Date: Fri, 15 Mar 2019 19:47:29 -0400 Subject: [PATCH 409/667] Fix EC2 children. --- ScoutSuite/providers/aws/resources/ec2/service.py | 4 ++-- ScoutSuite/providers/aws/resources/ec2/vpcs.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/ec2/service.py b/ScoutSuite/providers/aws/resources/ec2/service.py index 736a76b81..3258f298d 100644 --- a/ScoutSuite/providers/aws/resources/ec2/service.py +++ b/ScoutSuite/providers/aws/resources/ec2/service.py @@ -1,13 +1,13 @@ from ScoutSuite.providers.aws.resources.regions import Regions from ScoutSuite.providers.aws.resources.ec2.ami import AmazonMachineImages -from ScoutSuite.providers.aws.resources.ec2.vpcs import Vpcs +from ScoutSuite.providers.aws.resources.ec2.vpcs import Ec2Vpcs from ScoutSuite.providers.aws.resources.ec2.snapshots import Snapshots from ScoutSuite.providers.aws.resources.ec2.volumes import Volumes class EC2(Regions): _children = [ - (Vpcs, 'vpcs'), + (Ec2Vpcs, 'vpcs'), (AmazonMachineImages, 'images'), (Snapshots, 'snapshots'), (Volumes, 'volumes') diff --git a/ScoutSuite/providers/aws/resources/ec2/vpcs.py b/ScoutSuite/providers/aws/resources/ec2/vpcs.py index c3125df21..28ec102e4 100644 --- a/ScoutSuite/providers/aws/resources/ec2/vpcs.py +++ b/ScoutSuite/providers/aws/resources/ec2/vpcs.py @@ -11,4 +11,4 @@ class Ec2Vpcs(Vpcs): (EC2Instances, 'instances'), (SecurityGroups, 'security_groups'), (NetworkInterfaces, 'network_interfaces') - ] \ No newline at end of file + ] From 889d8e04f5c928135a38831305ef483b6eb2f907 Mon Sep 17 00:00:00 2001 From: misg Date: Fri, 15 Mar 2019 19:47:29 -0400 Subject: [PATCH 410/667] Fix EC2 children. --- ScoutSuite/providers/aws/resources/ec2/service.py | 4 ++-- ScoutSuite/providers/aws/resources/ec2/vpcs.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/ec2/service.py b/ScoutSuite/providers/aws/resources/ec2/service.py index 736a76b81..3258f298d 100644 --- a/ScoutSuite/providers/aws/resources/ec2/service.py +++ b/ScoutSuite/providers/aws/resources/ec2/service.py @@ -1,13 +1,13 @@ from ScoutSuite.providers.aws.resources.regions import Regions from ScoutSuite.providers.aws.resources.ec2.ami import AmazonMachineImages -from ScoutSuite.providers.aws.resources.ec2.vpcs import Vpcs +from ScoutSuite.providers.aws.resources.ec2.vpcs import Ec2Vpcs from ScoutSuite.providers.aws.resources.ec2.snapshots import Snapshots from ScoutSuite.providers.aws.resources.ec2.volumes import Volumes class EC2(Regions): _children = [ - (Vpcs, 'vpcs'), + (Ec2Vpcs, 'vpcs'), (AmazonMachineImages, 'images'), (Snapshots, 'snapshots'), (Volumes, 'volumes') diff --git a/ScoutSuite/providers/aws/resources/ec2/vpcs.py b/ScoutSuite/providers/aws/resources/ec2/vpcs.py index c3125df21..28ec102e4 100644 --- a/ScoutSuite/providers/aws/resources/ec2/vpcs.py +++ b/ScoutSuite/providers/aws/resources/ec2/vpcs.py @@ -11,4 +11,4 @@ class Ec2Vpcs(Vpcs): (EC2Instances, 'instances'), (SecurityGroups, 'security_groups'), (NetworkInterfaces, 'network_interfaces') - ] \ No newline at end of file + ] From 9756883510159f44de2f3f60364a53a495760c3e Mon Sep 17 00:00:00 2001 From: misg Date: Fri, 15 Mar 2019 20:29:38 -0400 Subject: [PATCH 411/667] Fix get_vpcs method. --- ScoutSuite/providers/aws/facade/ec2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/facade/ec2.py b/ScoutSuite/providers/aws/facade/ec2.py index 704b61358..f221eb6c8 100644 --- a/ScoutSuite/providers/aws/facade/ec2.py +++ b/ScoutSuite/providers/aws/facade/ec2.py @@ -36,7 +36,7 @@ async def get_security_groups(self, region, vpc): 'ec2', region, self.session, 'describe_security_groups', 'SecurityGroups', Filters=filters) async def get_vpcs(self, region): - ec2_client = await run_concurrently(lambda: boto3.client('ec2', region_name=region)) + ec2_client = AWSFacadeUtils.get_client('ec2', region, self.session) return await run_concurrently(lambda: ec2_client.describe_vpcs()['Vpcs']) async def get_images(self, region, owner_id): From 2c0c1a3083c0171c63d35835efab8871ce31b94e Mon Sep 17 00:00:00 2001 From: misg Date: Fri, 15 Mar 2019 20:29:38 -0400 Subject: [PATCH 412/667] Fix get_vpcs method. --- ScoutSuite/providers/aws/facade/ec2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/facade/ec2.py b/ScoutSuite/providers/aws/facade/ec2.py index 704b61358..f221eb6c8 100644 --- a/ScoutSuite/providers/aws/facade/ec2.py +++ b/ScoutSuite/providers/aws/facade/ec2.py @@ -36,7 +36,7 @@ async def get_security_groups(self, region, vpc): 'ec2', region, self.session, 'describe_security_groups', 'SecurityGroups', Filters=filters) async def get_vpcs(self, region): - ec2_client = await run_concurrently(lambda: boto3.client('ec2', region_name=region)) + ec2_client = AWSFacadeUtils.get_client('ec2', region, self.session) return await run_concurrently(lambda: ec2_client.describe_vpcs()['Vpcs']) async def get_images(self, region, owner_id): From 9ec611f3221f38f44075e7ba41381aa3fd610c92 Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 16 Mar 2019 10:29:06 -0400 Subject: [PATCH 413/667] Define a facade for ELB service. --- ScoutSuite/providers/aws/facade/elb.py | 35 ++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 ScoutSuite/providers/aws/facade/elb.py diff --git a/ScoutSuite/providers/aws/facade/elb.py b/ScoutSuite/providers/aws/facade/elb.py new file mode 100644 index 000000000..9523b2ae5 --- /dev/null +++ b/ScoutSuite/providers/aws/facade/elb.py @@ -0,0 +1,35 @@ +from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils +from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade +from ScoutSuite.providers.aws.utils import ec2_classic +from ScoutSuite.providers.utils import run_concurrently + +from asyncio import Lock + + +class ELBFacade(AWSBaseFacade): + regional_load_balancers_cache_locks = {} + load_balancers_cache = {} + + async def get_load_balancers(self, region: str, vpc: str): + await self.cache_load_balancers(region) + return [load_balancer for load_balancer in self.load_balancers_cache[region] if load_balancer['VpcId'] == vpc] + + async def cache_load_balancers(self, region): + async with self.regional_load_balancers_cache_locks.setdefault(region, Lock()): + if region in self.load_balancers_cache: + return + + self.load_balancers_cache[region] =\ + await AWSFacadeUtils.get_all_pages('elb', region, self.session, + 'describe_load_balancers', 'LoadBalancerDescriptions') + + for load_balancer in self.load_balancers_cache[region]: + load_balancer['VpcId'] =\ + load_balancer['VPCId'] if 'VPCId' in load_balancer and load_balancer['VPCId'] else ec2_classic + + async def get_load_balancer_attributes(self, region:str, load_balancer: str): + elb_client = AWSFacadeUtils.get_client('elb', region, self.session) + return await run_concurrently( + lambda: elb_client.describe_load_balancer_attributes( + LoadBalancerName=load_balancer)['LoadBalancerAttributes'] + ) From 83fdc116a8b2eb80932aab7a0247a3daf49dbf79 Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 16 Mar 2019 10:29:27 -0400 Subject: [PATCH 414/667] Add ELB facade to AWS facade. --- ScoutSuite/providers/aws/facade/facade.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/facade/facade.py b/ScoutSuite/providers/aws/facade/facade.py index 92d09fa63..a37542fc9 100644 --- a/ScoutSuite/providers/aws/facade/facade.py +++ b/ScoutSuite/providers/aws/facade/facade.py @@ -12,8 +12,8 @@ from ScoutSuite.providers.aws.facade.elasticache import ElastiCacheFacade from ScoutSuite.providers.aws.facade.emr import EMRFacade from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade +from ScoutSuite.providers.aws.facade.elb import ELBFacade from ScoutSuite.providers.utils import run_concurrently -from ScoutSuite.core.console import print_error, print_debug class AWSFacade(AWSBaseFacade): @@ -29,6 +29,7 @@ def __init__(self, credentials: dict = None): self.efs = EFSFacade(self.session) self.elasticache = ElastiCacheFacade(self.session) self.emr = EMRFacade(self.session) + self.elb = ELBFacade(self.session) async def build_region_list(self, service: str, chosen_regions=None, partition_name='aws'): service = 'ec2containerservice' if service == 'ecs' else service @@ -67,3 +68,4 @@ def _set_session(self, credentials: dict): self.efs = EFSFacade(self.session) self.elasticache = ElastiCacheFacade(self.session) self.emr = EMRFacade(self.session) + self.elb = ELBFacade(self.session) From 02129ef83338afc854b5740620c6eeffccdde4d4 Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 16 Mar 2019 10:29:46 -0400 Subject: [PATCH 415/667] Define LoadBalancers resources. --- .../aws/resources/elb/load_balancers.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 ScoutSuite/providers/aws/resources/elb/load_balancers.py diff --git a/ScoutSuite/providers/aws/resources/elb/load_balancers.py b/ScoutSuite/providers/aws/resources/elb/load_balancers.py new file mode 100644 index 000000000..17d90897e --- /dev/null +++ b/ScoutSuite/providers/aws/resources/elb/load_balancers.py @@ -0,0 +1,34 @@ +from ScoutSuite.providers.aws.resources.resources import AWSResources +from ScoutSuite.providers.aws.utils import get_keys +from ScoutSuite.providers.utils import get_non_provider_id + + +class LoadBalancers(AWSResources): + async def fetch_all(self, **kwargs): + raw_load_balancers = await self.facade.elb.get_load_balancers(self.scope['region'], self.scope['vpc']) + # TODO: parse is async, parallelize the following loop: + for raw_load_balancer in raw_load_balancers: + id, load_balancer = await self._parse_load_balancer(raw_load_balancer) + self[id] = load_balancer + + async def _parse_load_balancer(self, raw_load_balancer): + load_balancer = {'name': raw_load_balancer.pop('LoadBalancerName')} + get_keys(raw_load_balancer, load_balancer, ['DNSName', 'CreatedTime', 'AvailabilityZones', 'Subnets', 'Scheme']) + + load_balancer['security_groups'] = [] + for sg in raw_load_balancer['SecurityGroups']: + load_balancer['security_groups'].append({'GroupId': sg}) + + load_balancer['listeners'] = {} + for l in raw_load_balancer['ListenerDescriptions']: + listener = l['Listener'] + load_balancer['listeners'][l['Listener']['LoadBalancerPort']] = listener + + load_balancer['instances'] = [] + for i in raw_load_balancer['Instances']: + load_balancer['instances'].append(i['InstanceId']) + + load_balancer['attributes'] =\ + await self.facade.elb.get_load_balancer_attributes(self.scope['region'], load_balancer['name']) + + return get_non_provider_id(load_balancer['name']), load_balancer From 86e1a2bdd6229cee9da9569d72f1454d328d62ce Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 16 Mar 2019 10:30:09 -0400 Subject: [PATCH 416/667] Define ELBVpcs resources. --- ScoutSuite/providers/aws/resources/elb/vpcs.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 ScoutSuite/providers/aws/resources/elb/vpcs.py diff --git a/ScoutSuite/providers/aws/resources/elb/vpcs.py b/ScoutSuite/providers/aws/resources/elb/vpcs.py new file mode 100644 index 000000000..dfced75c9 --- /dev/null +++ b/ScoutSuite/providers/aws/resources/elb/vpcs.py @@ -0,0 +1,11 @@ +from ScoutSuite.providers.aws.resources.vpcs import Vpcs +from .load_balancers import LoadBalancers + + +class ELBVpcs(Vpcs): + _children = [ + (LoadBalancers, 'elbs'), + ] + + def __init__(self, facade, scope: dict): + super(ELBVpcs, self).__init__(facade, scope, add_ec2_classic=True) From 86b564339ea47727d3fdcdc93a5b24733a111f7a Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 16 Mar 2019 10:30:46 -0400 Subject: [PATCH 417/667] Define ELB service. --- ScoutSuite/providers/aws/resources/elb/service.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 ScoutSuite/providers/aws/resources/elb/service.py diff --git a/ScoutSuite/providers/aws/resources/elb/service.py b/ScoutSuite/providers/aws/resources/elb/service.py new file mode 100644 index 000000000..dd61c31fd --- /dev/null +++ b/ScoutSuite/providers/aws/resources/elb/service.py @@ -0,0 +1,12 @@ +from ScoutSuite.providers.aws.resources.regions import Regions + +from .vpcs import ELBVpcs + + +class ELB(Regions): + _children = [ + (ELBVpcs, 'vpcs') + ] + + def __init__(self): + super(ELB, self).__init__('elb') From 93e475057ba5eece771ecd61cae1507744177809 Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 16 Mar 2019 10:31:34 -0400 Subject: [PATCH 418/667] Migrate ELB service to the new architecture. --- ScoutSuite/providers/aws/configs/services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index 2c5311aef..e0abf971e 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -7,8 +7,8 @@ from ScoutSuite.providers.aws.resources.directconnect.service import DirectConnect from ScoutSuite.providers.aws.resources.ec2.service import EC2 from ScoutSuite.providers.aws.resources.efs.service import EFS +from ScoutSuite.providers.aws.resources.elb.service import ELB from ScoutSuite.providers.aws.resources.elasticache.service import ElastiCache -from ScoutSuite.providers.aws.services.elb import ELBConfig from ScoutSuite.providers.aws.services.elbv2 import ELBv2Config from ScoutSuite.providers.aws.resources.emr.service import EMR from ScoutSuite.providers.aws.services.iam import IAMConfig @@ -61,7 +61,7 @@ def __init__(self, metadata=None, thread_config=4, **kwargs): self.ec2 = EC2() self.efs = EFS() self.elasticache = ElastiCache() - self.elb = ELBConfig(metadata['compute']['elb'], thread_config) + self.elb = ELB() self.elbv2 = ELBv2Config(metadata['compute']['elbv2'], thread_config) self.emr = EMR() self.iam = IAMConfig(thread_config) From dda3b70eb29b76c5594e47ac68fef7fa9fdb133a Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 16 Mar 2019 10:31:53 -0400 Subject: [PATCH 419/667] Remove old ELB service support. --- ScoutSuite/providers/aws/services/elb.py | 79 ------------------------ 1 file changed, 79 deletions(-) delete mode 100644 ScoutSuite/providers/aws/services/elb.py diff --git a/ScoutSuite/providers/aws/services/elb.py b/ScoutSuite/providers/aws/services/elb.py deleted file mode 100644 index 8edb73c62..000000000 --- a/ScoutSuite/providers/aws/services/elb.py +++ /dev/null @@ -1,79 +0,0 @@ -# -*- coding: utf-8 -*- -""" -ELB-related classes and functions -""" - -from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients -from ScoutSuite.providers.aws.configs.vpc import VPCConfig -from ScoutSuite.utils import manage_dictionary -from ScoutSuite.providers.aws.utils import ec2_classic, get_keys - - -######################################## -# ELBRegionConfig -######################################## - -class ELBRegionConfig(RegionConfig): - """ - ELB configuration for a single AWS region - """ - elb_policies = {} - - def parse_elb(self, global_params, region, lb): - """ - - :param lb: - :param global_params: - :param region: Name of the AWS region - :return: - """ - elb = {'name': lb.pop('LoadBalancerName')} - vpc_id = lb['VPCId'] if 'VPCId' in lb and lb['VPCId'] else ec2_classic - manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types)) - get_keys(lb, elb, ['DNSName', 'CreatedTime', 'AvailabilityZones', 'Subnets', 'Scheme']) - elb['security_groups'] = [] - for sg in lb['SecurityGroups']: - elb['security_groups'].append({'GroupId': sg}) - manage_dictionary(elb, 'listeners', {}) - policy_names = [] - for l in lb['ListenerDescriptions']: - listener = l['Listener'] - manage_dictionary(listener, 'policies', []) - for policy_name in l['PolicyNames']: - policy_id = self.get_non_provider_id(policy_name) - listener['policies'].append(policy_id) - if policy_id not in self.elb_policies: - policy_names.append(policy_name) - elb['listeners'][l['Listener']['LoadBalancerPort']] = listener - # Fetch LB policies here. This is not ideal, but the alternative is to download all policies and clean up - # after... - if len(policy_names): - policies = \ - api_clients[region].describe_load_balancer_policies(LoadBalancerName=elb['name'], - PolicyNames=policy_names)['PolicyDescriptions'] - for policy in policies: - policy['name'] = policy.pop('PolicyName') - policy_id = self.get_non_provider_id(policy['name']) - self.elb_policies[policy_id] = policy - manage_dictionary(elb, 'instances', []) - for i in lb['Instances']: - elb['instances'].append(i['InstanceId']) - # Get attributes - elb['attributes'] = api_clients[region].describe_load_balancer_attributes(LoadBalancerName=elb['name'])[ - 'LoadBalancerAttributes'] - self.vpcs[vpc_id].elbs[self.get_non_provider_id(elb['name'])] = elb - - -######################################## -# ELBConfig -######################################## - -class ELBConfig(RegionalServiceConfig): - """ - ELB configuration for all AWS regions - """ - - region_config_class = ELBRegionConfig - - def __init__(self, service_metadata, thread_config=4): - super(ELBConfig, self).__init__(service_metadata, thread_config) From d7fa7820edd74a60162b5bbe3e25924bed9fd2c2 Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 16 Mar 2019 10:45:56 -0400 Subject: [PATCH 420/667] Remove old services. --- .../providers/azure/services/__init__.py | 0 .../providers/azure/services/keyvault.py | 27 ---- .../providers/azure/services/monitor.py | 40 ----- .../providers/azure/services/network.py | 145 ------------------ .../azure/services/securitycenter.py | 51 ------ .../providers/azure/services/sqldatabase.py | 112 -------------- .../azure/services/storageaccounts.py | 59 ------- 7 files changed, 434 deletions(-) delete mode 100644 ScoutSuite/providers/azure/services/__init__.py delete mode 100644 ScoutSuite/providers/azure/services/keyvault.py delete mode 100644 ScoutSuite/providers/azure/services/monitor.py delete mode 100644 ScoutSuite/providers/azure/services/network.py delete mode 100644 ScoutSuite/providers/azure/services/securitycenter.py delete mode 100644 ScoutSuite/providers/azure/services/sqldatabase.py delete mode 100644 ScoutSuite/providers/azure/services/storageaccounts.py diff --git a/ScoutSuite/providers/azure/services/__init__.py b/ScoutSuite/providers/azure/services/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/ScoutSuite/providers/azure/services/keyvault.py b/ScoutSuite/providers/azure/services/keyvault.py deleted file mode 100644 index 1bf14c6e1..000000000 --- a/ScoutSuite/providers/azure/services/keyvault.py +++ /dev/null @@ -1,27 +0,0 @@ -# -*- coding: utf-8 -*- - -from ScoutSuite.providers.azure.configs.base import AzureBaseConfig - - -class KeyVaultConfig(AzureBaseConfig): - targets = ( - ('vaults', 'Key Vaults', 'list_by_subscription', {}, False), - ) - - def __init__(self, thread_config): - - self.vaults = {} - self.vaults_count = 0 - - super(KeyVaultConfig, self).__init__(thread_config) - - def parse_vaults(self, vault, params): - vault_dict = {} - vault_dict['id'] = self.get_non_provider_id(vault.id) - vault_dict['name'] = vault.name - vault_dict['public_access_allowed'] = self._is_public_access_allowed(vault) - - self.vaults[vault_dict['id']] = vault_dict - - def _is_public_access_allowed(self, vault): - return vault.properties.network_acls is None diff --git a/ScoutSuite/providers/azure/services/monitor.py b/ScoutSuite/providers/azure/services/monitor.py deleted file mode 100644 index 6d4514cf0..000000000 --- a/ScoutSuite/providers/azure/services/monitor.py +++ /dev/null @@ -1,40 +0,0 @@ -# -*- coding: utf-8 -*- - -import datetime - -from ScoutSuite.providers.azure.configs.base import AzureBaseConfig - - -class MonitorConfig(AzureBaseConfig): - time_format = "%Y-%m-%dT%H:%M:%S.%f" - utc_now = datetime.datetime.utcnow() - end_time = utc_now.strftime(time_format) - timespan = datetime.timedelta(90) # 90 days of timespan - start_time = (utc_now - timespan).strftime(time_format) - - targets = ( - ('activity_logs', 'Activity Logs', 'list', - {'filter': "eventTimestamp ge {} and eventTimestamp le {}".format(start_time, end_time)}, - False), - ) - - def __init__(self, thread_config): - - self.activity_logs = {} - self.activity_logs['storage_accounts'] = {} - - super(MonitorConfig, self).__init__(thread_config) - - def parse_activity_logs(self, activity_log, params): - if activity_log.resource_type.value == 'Microsoft.Storage/storageAccounts': - self._parse_storage_account_logs(activity_log) - - def _parse_storage_account_logs(self, activity_log): - storage_account_id = self.get_non_provider_id(activity_log.resource_id.lower()) - - if storage_account_id not in self.activity_logs['storage_accounts']: - self.activity_logs['storage_accounts'][storage_account_id] =\ - {'access_keys_rotated': False} - - if activity_log.operation_name.value == 'Microsoft.Storage/storageAccounts/regenerateKey/action': - self.activity_logs['storage_accounts'][storage_account_id]['access_keys_rotated'] = True diff --git a/ScoutSuite/providers/azure/services/network.py b/ScoutSuite/providers/azure/services/network.py deleted file mode 100644 index a23bb0a9b..000000000 --- a/ScoutSuite/providers/azure/services/network.py +++ /dev/null @@ -1,145 +0,0 @@ -# -*- coding: utf-8 -*- - -from ScoutSuite.providers.azure.configs.base import AzureBaseConfig - - -class NetworkConfig(AzureBaseConfig): - targets = ( - ('network_watchers', 'Network Watchers', 'list_all', {}, False), - ('network_security_groups', 'Network Security Group', 'list_all', {}, False), - ) - - def __init__(self, thread_config): - self.network_watchers = {} - self.network_watchers_count = 0 - - self.network_security_groups = {} - self.network_security_groups_count = 0 - - super(NetworkConfig, self).__init__(thread_config) - - def parse_network_watchers(self, network_watcher, params): - network_watcher_dict = {} - network_watcher_dict['id'] = network_watcher.id - network_watcher_dict['name'] = network_watcher.name - network_watcher_dict['provisioning_state'] = network_watcher.provisioning_state - network_watcher_dict['location'] = network_watcher.location - network_watcher_dict['etag'] = network_watcher.etag - - self.network_watchers[network_watcher_dict['id']] = network_watcher_dict - - def parse_network_security_groups(self, network_security_group, params): - network_security_group_dict = {} - network_security_group_dict['id'] = network_security_group.id - network_security_group_dict['name'] = network_security_group.name - network_security_group_dict['provisioning_state'] = network_security_group.provisioning_state - network_security_group_dict['location'] = network_security_group.location - network_security_group_dict['resource_guid'] = network_security_group.resource_guid - network_security_group_dict['etag'] = network_security_group.etag - - network_security_group_dict['security_rules'] = self._parse_security_rules(network_security_group) - - exposed_ports = self._parse_exposed_ports(network_security_group) - network_security_group_dict['exposed_ports'] = exposed_ports - network_security_group_dict['exposed_port_ranges'] = self._format_ports(exposed_ports) - - self.network_security_groups[network_security_group_dict['id']] = network_security_group_dict - - def _parse_security_rules(self, network_security_group): - security_rules = {} - for sr in network_security_group.security_rules: - security_rule_dict = {} - security_rule_dict['id'] = sr.id - security_rule_dict['name'] = sr.name - security_rule_dict['allow'] = sr.access == "Allow" - security_rule_dict['priority'] = sr.priority - security_rule_dict['description'] = sr.description - security_rule_dict['provisioning_state'] = sr.provisioning_state - - security_rule_dict['protocol'] = sr.protocol - security_rule_dict['direction'] = sr.direction - - source_address_prefixes = self._merge_prefixes_or_ports(sr.source_address_prefix, - sr.source_address_prefixes) - security_rule_dict['source_address_prefixes'] = source_address_prefixes - - source_port_ranges = self._merge_prefixes_or_ports(sr.source_port_range, sr.source_port_ranges) - security_rule_dict['source_port_ranges'] = source_port_ranges - security_rule_dict['source_ports'] = self._parse_ports(source_port_ranges) - - destination_address_prefixes = self._merge_prefixes_or_ports(sr.destination_address_prefix, - sr.destination_address_prefixes) - security_rule_dict['destination_address_prefixes'] = destination_address_prefixes - - destination_port_ranges = self._merge_prefixes_or_ports(sr.destination_port_range, - sr.destination_port_ranges) - security_rule_dict['destination_port_ranges'] = destination_port_ranges - security_rule_dict['destination_ports'] = self._parse_ports(destination_port_ranges) - - security_rule_dict['etag'] = sr.etag - - security_rules[security_rule_dict['id']] = security_rule_dict - - return security_rules - - def _parse_ports(self, port_ranges): - ports = set() - for pr in port_ranges: - if pr == "*": - for p in range(0, 65535 + 1): - ports.add(p) - break - elif "-" in pr: - lower, upper = pr.split("-") - for p in range(int(lower), int(upper) + 1): - ports.add(p) - else: - ports.add(int(pr)) - ports = list(ports) - ports.sort() - return ports - - def _parse_exposed_ports(self, network_security_group): - exposed_ports = set() - - # Sort by priority. - rules = network_security_group.default_security_rules + network_security_group.security_rules - rules.sort(key=lambda x: x.priority, reverse=True) - - for sr in rules: - if sr.direction == "Inbound" and (sr.source_address_prefix == "*" - or sr.source_address_prefix == "Internet"): - port_ranges = self._merge_prefixes_or_ports(sr.destination_port_range, - sr.destination_port_ranges) - ports = self._parse_ports(port_ranges) - if sr.access == "Allow": - for p in ports: - exposed_ports.add(p) - else: - for p in ports: - exposed_ports.discard(p) - exposed_ports = list(exposed_ports) - exposed_ports.sort() - return exposed_ports - - def _merge_prefixes_or_ports(self, port_range, port_ranges): - port_ranges = port_ranges if port_ranges else [] - if port_range: - port_ranges.append(port_range) - return port_ranges - - def _format_ports(self, ports): - port_ranges = [] - start = None - for i in range(0, 65535 + 1): - if i in ports: - if not start: - start = i - else: - if start: - if i - 1 == start: - port_ranges.append(str(start)) - else: - port_ranges.append(str(start) + "-" + str(i - 1)) - start = None - return port_ranges diff --git a/ScoutSuite/providers/azure/services/securitycenter.py b/ScoutSuite/providers/azure/services/securitycenter.py deleted file mode 100644 index af2865eb3..000000000 --- a/ScoutSuite/providers/azure/services/securitycenter.py +++ /dev/null @@ -1,51 +0,0 @@ -# -*- coding: utf-8 -*- - -from ScoutSuite.providers.azure.configs.base import AzureBaseConfig - - -class SecurityCenterConfig(AzureBaseConfig): - targets = ( - ('pricings', 'Pricings', 'list', {}, False), - ('security_contacts', 'Security Contacts', 'list', {}, False), - ('auto_provisioning_settings', 'Auto Provisioning Settings', 'list', {}, False) - ) - - def __init__(self, thread_config): - self.pricings = {} - self.pricings_count = 0 - - self.security_contacts = {} - self.security_contacts_count = 0 - - self.auto_provisioning_settings = {} - self.auto_provisioning_settings_count = 0 - - super(SecurityCenterConfig, self).__init__(thread_config) - - def parse_pricings(self, pricing, params): - pricing_dict = {} - pricing_dict['id'] = pricing.id - pricing_dict['name'] = pricing.name - pricing_dict['pricing_tier'] = pricing.pricing_tier - - self.pricings[pricing_dict['id']] = pricing_dict - - def parse_security_contacts(self, security_contact, params): - security_contact_dict = {} - security_contact_dict['id'] = security_contact.id - security_contact_dict['name'] = security_contact.name - security_contact_dict['email'] = security_contact.email - security_contact_dict['phone'] = security_contact.phone - security_contact_dict['alert_notifications'] = security_contact.alert_notifications == "On" - security_contact_dict['alerts_to_admins'] = security_contact.alerts_to_admins == "On" - security_contact_dict['additional_properties'] = security_contact.additional_properties - - self.security_contacts[security_contact_dict['id']] = security_contact_dict - - def parse_auto_provisioning_settings(self, auto_provisioning_setting, params): - auto_provisioning_setting_dict = {} - auto_provisioning_setting_dict['id'] = auto_provisioning_setting.id - auto_provisioning_setting_dict['name'] = auto_provisioning_setting.name - auto_provisioning_setting_dict['auto_provision'] = auto_provisioning_setting.auto_provision - - self.auto_provisioning_settings[auto_provisioning_setting_dict['id']] = auto_provisioning_setting_dict diff --git a/ScoutSuite/providers/azure/services/sqldatabase.py b/ScoutSuite/providers/azure/services/sqldatabase.py deleted file mode 100644 index cfec0ffe9..000000000 --- a/ScoutSuite/providers/azure/services/sqldatabase.py +++ /dev/null @@ -1,112 +0,0 @@ -# -*- coding: utf-8 -*- - -from ScoutSuite.providers.azure.configs.base import AzureBaseConfig -from ScoutSuite.providers.azure.utils import get_resource_group_name -from msrestazure.azure_exceptions import CloudError - -class SQLDatabaseConfig(AzureBaseConfig): - targets = ( - ('servers', 'Servers', 'list', {}, False), - ) - - def __init__(self, thread_config): - - self.servers = {} - self.servers_count = 0 - - super(SQLDatabaseConfig, self).__init__(thread_config) - - def parse_servers(self, server, params): - server_dict = {} - server_dict['id'] = self.get_non_provider_id(server.id) - server_dict['name'] = server.name - server_dict['ad_admin_configured'] = self._is_ad_admin_configured(server) - server_dict['databases'] = self._parse_databases(server) - - self.servers[server_dict['id']] = server_dict - - def _is_ad_admin_configured(self, server): - return server.azure_ad_admin_settings is not None - - def _parse_databases(self, server): - databases = {} - for db in server.databases: - # Azure automatically creates a reserved read-only 'master' database to manage an SQL server, ignore it: - if db.name == "master": - continue - - db_dict = {} - db_dict['id'] = db.name - db_dict['auditing_enabled'] = self._is_auditing_enabled(db) - db_dict['threat_detection_enabled'] = self._is_threat_detection_enabled(db) - db_dict['transparent_data_encryption_enabled'] = self._is_transparent_data_encryption_enabled(db) - db_dict['replication_configured'] = self._is_replication_configured(db) - databases[db.name] = db_dict - - return databases - - def _is_auditing_enabled(self, db): - return db.auditing_settings.state == "Enabled" - - def _is_threat_detection_enabled(self, db): - return db.threat_detection_settings.state == "Enabled" - - def _is_transparent_data_encryption_enabled(self, db): - return db.transparent_data_encryption_settings.status == "Enabled" - - def _is_replication_configured(self, db): - return len(db.replication_links) > 0 - - def _get_targets(self, response_attribute, api_client, method, list_params, ignore_list_error): - if response_attribute == "Servers": - return self._get_servers(api_client, method, list_params) - else: - return super(SQLDatabaseConfig, self)._get_targets(response_attribute, api_client, method, - list_params, ignore_list_error) - - def _get_servers(self, api_client, method, list_params): - servers = [] - servers_raw = method(**list_params) - for server in servers_raw: - resource_group_name = get_resource_group_name(server.id) - setattr(server, "azure_ad_admin_settings", - self._get_azure_ad_admin_settings(api_client, resource_group_name, server.name)) - setattr(server, "databases", - self._get_databases(api_client, resource_group_name, server.name)) - servers.append(server) - - return servers - - def _get_databases(self, api_client, resource_group_name, server_name): - databases = [] - databases_raw = api_client.databases.list_by_server(resource_group_name, server_name) - for db in databases_raw: - setattr(db, "auditing_settings", - self._get_auditing_settings(api_client, resource_group_name, server_name, db.name)) - setattr(db, "threat_detection_settings", - self._get_threat_detection_settings(api_client, resource_group_name, server_name, db.name)) - setattr(db, "transparent_data_encryption_settings", - self._get_transparent_data_encryption_settings(api_client, resource_group_name, server_name, db.name)) - setattr(db, "replication_links", - list(self._get_replication_links(api_client, resource_group_name, server_name, db.name))) - databases.append(db) - - return databases - - def _get_auditing_settings(self, api_client, resource_group_name, server_name, database_name): - return api_client.database_blob_auditing_policies.get(resource_group_name, server_name, database_name) - - def _get_threat_detection_settings(self, api_client, resource_group_name, server_name, database_name): - return api_client.database_threat_detection_policies.get(resource_group_name, server_name, database_name) - - def _get_transparent_data_encryption_settings(self, api_client, resource_group_name, server_name, database_name): - return api_client.transparent_data_encryptions.get(resource_group_name, server_name, database_name) - - def _get_azure_ad_admin_settings(self, api_client, resource_group_name, server_name): - try: - return api_client.server_azure_ad_administrators.get(resource_group_name, server_name) - except CloudError: # no ad admin configured returns a 404 error - return None - - def _get_replication_links(self, api_client, resource_group_name, server_name, database_name): - return api_client.replication_links.list_by_database(resource_group_name, server_name, database_name) diff --git a/ScoutSuite/providers/azure/services/storageaccounts.py b/ScoutSuite/providers/azure/services/storageaccounts.py deleted file mode 100644 index db57c0a65..000000000 --- a/ScoutSuite/providers/azure/services/storageaccounts.py +++ /dev/null @@ -1,59 +0,0 @@ -# -*- coding: utf-8 -*- - -from ScoutSuite.providers.azure.configs.base import AzureBaseConfig -from ScoutSuite.providers.azure.utils import get_resource_group_name - -class StorageAccountsConfig(AzureBaseConfig): - targets = ( - ('storage_accounts', 'Storage Accounts', 'list', {}, False), - ) - - def __init__(self, thread_config): - - self.storage_accounts = {} - self.storage_accounts_count = 0 - - super(StorageAccountsConfig, self).__init__(thread_config) - - def parse_storage_accounts(self, storage_account, params): - - storage_account_dict = {} - - storage_account_dict['id'] = self.get_non_provider_id(storage_account.id.lower()) - storage_account_dict['name'] = storage_account.name - storage_account_dict['https_traffic_enabled'] = storage_account.enable_https_traffic_only - storage_account_dict['public_traffic_allowed'] = self._is_public_traffic_allowed(storage_account) - storage_account_dict['blob_containers'] = self._parse_blob_containers(storage_account) - - self.storage_accounts[storage_account_dict['id']] = storage_account_dict - - def _is_public_traffic_allowed(self, storage_account): - return storage_account.network_rule_set.default_action == "Allow" - - def _parse_blob_containers(self, storage_account): - blob_containers = {} - for container in storage_account.blob_containers: - container_dict = {} - container_dict['id'] = container.name - container_dict['public_access_allowed'] = container.public_access != "None" - blob_containers[container.name] = container_dict - - return blob_containers - - def _get_targets(self, response_attribute, api_client, method, list_params, ignore_list_error): - if response_attribute == "Storage Accounts": - return self._get_storage_accounts(api_client, method, list_params) - else: - return super(StorageAccountsConfig, self)._get_targets(response_attribute, api_client, method, - list_params, ignore_list_error) - - def _get_storage_accounts(self, api_client, method, list_params): - storage_accounts = [] - storage_accounts_raw = method(**list_params) - for storage_account in storage_accounts_raw: - resource_group_name = get_resource_group_name(storage_account.id) - setattr(storage_account, "blob_containers", \ - api_client.blob_containers.list(resource_group_name, storage_account.name).value) - storage_accounts.append(storage_account) - - return storage_accounts From 5baa3d1ae4f78a95fe45fb9811d79c7547e11e2e Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 16 Mar 2019 11:52:07 -0400 Subject: [PATCH 421/667] Fix imports. --- ScoutSuite/providers/aws/facade/efs.py | 10 ++++++---- ScoutSuite/providers/aws/resources/ec2/instances.py | 2 +- ScoutSuite/providers/aws/resources/ec2/snapshots.py | 2 +- ScoutSuite/providers/aws/resources/ec2/volumes.py | 2 +- ScoutSuite/providers/aws/resources/regions.py | 2 +- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/efs.py b/ScoutSuite/providers/aws/facade/efs.py index b33955d89..25d2b4db7 100644 --- a/ScoutSuite/providers/aws/facade/efs.py +++ b/ScoutSuite/providers/aws/facade/efs.py @@ -1,20 +1,22 @@ from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils -from ScoutSuite.providers.aws.aws import handle_truncated_response from ScoutSuite.providers.utils import run_concurrently from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade class EFSFacade(AWSBaseFacade): async def get_file_systems(self, region: str): - file_systems = await AWSFacadeUtils.get_all_pages('efs', region, self.session, 'describe_file_systems', 'FileSystems') + file_systems = await AWSFacadeUtils.get_all_pages( + 'efs', region, self.session, 'describe_file_systems', 'FileSystems') client = AWSFacadeUtils.get_client('efs', region, self.session) for file_system in file_systems: file_system_id = file_system['FileSystemId'] - file_system['Tags'] = await run_concurrently(lambda: client.describe_tags(FileSystemId=file_system_id)['Tags']) + file_system['Tags'] = await run_concurrently( + lambda: client.describe_tags(FileSystemId=file_system_id)['Tags']) # Get mount targets - mount_targets = await AWSFacadeUtils.get_all_pages('efs', region, self.session, 'describe_mount_targets', 'MountTargets', FileSystemId=file_system_id) + mount_targets = await AWSFacadeUtils.get_all_pages( + 'efs', region, self.session, 'describe_mount_targets', 'MountTargets', FileSystemId=file_system_id) file_system['MountTargets'] = {} for mount_target in mount_targets: mount_target_id = mount_target['MountTargetId'] diff --git a/ScoutSuite/providers/aws/resources/ec2/instances.py b/ScoutSuite/providers/aws/resources/ec2/instances.py index ba65a4164..ef999f427 100644 --- a/ScoutSuite/providers/aws/resources/ec2/instances.py +++ b/ScoutSuite/providers/aws/resources/ec2/instances.py @@ -1,5 +1,5 @@ from ScoutSuite.providers.aws.resources.resources import AWSResources -from ScoutSuite.providers.aws.aws import get_name +from ScoutSuite.providers.aws.utils import get_name from ScoutSuite.providers.aws.utils import get_keys import re diff --git a/ScoutSuite/providers/aws/resources/ec2/snapshots.py b/ScoutSuite/providers/aws/resources/ec2/snapshots.py index 9c36ca5ce..9fec72221 100644 --- a/ScoutSuite/providers/aws/resources/ec2/snapshots.py +++ b/ScoutSuite/providers/aws/resources/ec2/snapshots.py @@ -1,5 +1,5 @@ from ScoutSuite.providers.aws.resources.resources import AWSResources -from ScoutSuite.providers.aws.aws import get_name +from ScoutSuite.providers.aws.utils import get_name class Snapshots(AWSResources): diff --git a/ScoutSuite/providers/aws/resources/ec2/volumes.py b/ScoutSuite/providers/aws/resources/ec2/volumes.py index c36367b8a..9cdd02553 100644 --- a/ScoutSuite/providers/aws/resources/ec2/volumes.py +++ b/ScoutSuite/providers/aws/resources/ec2/volumes.py @@ -1,5 +1,5 @@ from ScoutSuite.providers.aws.resources.resources import AWSResources -from ScoutSuite.providers.aws.aws import get_name +from ScoutSuite.providers.aws.utils import get_name class Volumes(AWSResources): diff --git a/ScoutSuite/providers/aws/resources/regions.py b/ScoutSuite/providers/aws/resources/regions.py index d05227b1f..484a1404b 100644 --- a/ScoutSuite/providers/aws/resources/regions.py +++ b/ScoutSuite/providers/aws/resources/regions.py @@ -1,7 +1,7 @@ import abc import asyncio -from ScoutSuite.providers.aws.aws import get_aws_account_id +from ScoutSuite.providers.aws.utils import get_aws_account_id from ScoutSuite.providers.aws.resources.resources import AWSCompositeResources from ScoutSuite.providers.aws.facade.facade import AWSFacade From 4ba562be8b0a25b693ac4ccdc6959ac124daa7f5 Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 16 Mar 2019 22:56:07 -0400 Subject: [PATCH 422/667] Define a facade for ELBv2 service. --- ScoutSuite/providers/aws/facade/elbv2.py | 39 ++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 ScoutSuite/providers/aws/facade/elbv2.py diff --git a/ScoutSuite/providers/aws/facade/elbv2.py b/ScoutSuite/providers/aws/facade/elbv2.py new file mode 100644 index 000000000..e407089f8 --- /dev/null +++ b/ScoutSuite/providers/aws/facade/elbv2.py @@ -0,0 +1,39 @@ +from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils +from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade +from ScoutSuite.providers.aws.utils import ec2_classic +from ScoutSuite.providers.utils import run_concurrently + +from asyncio import Lock + + +class ELBv2Facade(AWSBaseFacade): + regional_load_balancers_cache_locks = {} + load_balancers_cache = {} + + async def get_load_balancers(self, region: str, vpc: str): + await self.cache_load_balancers(region) + return [load_balancer for load_balancer in self.load_balancers_cache[region] if load_balancer['VpcId'] == vpc] + + async def cache_load_balancers(self, region): + async with self.regional_load_balancers_cache_locks.setdefault(region, Lock()): + if region in self.load_balancers_cache: + return + + self.load_balancers_cache[region] =\ + await AWSFacadeUtils.get_all_pages('elbv2', region, self.session, + 'describe_load_balancers', 'LoadBalancers') + + for load_balancer in self.load_balancers_cache[region]: + load_balancer['VpcId'] =\ + load_balancer['VpcId'] if 'VpcId' in load_balancer and load_balancer['VpcId'] else ec2_classic + + async def get_load_balancer_attributes(self, region: str, load_balancer_arn: str): + elbv2_client = AWSFacadeUtils.get_client('elbv2', region, self.session) + return await run_concurrently( + lambda: elbv2_client.describe_load_balancer_attributes( + LoadBalancerArn=load_balancer_arn)['Attributes'] + ) + + async def get_listeners(self, region: str, load_balancer_arn: str): + return await AWSFacadeUtils.get_all_pages( + 'elbv2', region, self.session, 'describe_listeners', 'Listeners', LoadBalancerArn=load_balancer_arn) From 2efa154c71ac800ed5440c349302c4561f97726a Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 16 Mar 2019 22:56:28 -0400 Subject: [PATCH 423/667] Add ELBv2 facade to AWS facade. --- ScoutSuite/providers/aws/facade/facade.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/facade/facade.py b/ScoutSuite/providers/aws/facade/facade.py index 92d09fa63..3b519d799 100644 --- a/ScoutSuite/providers/aws/facade/facade.py +++ b/ScoutSuite/providers/aws/facade/facade.py @@ -11,9 +11,9 @@ from ScoutSuite.providers.aws.facade.efs import EFSFacade from ScoutSuite.providers.aws.facade.elasticache import ElastiCacheFacade from ScoutSuite.providers.aws.facade.emr import EMRFacade +from ScoutSuite.providers.aws.facade.elbv2 import ELBv2Facade from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade from ScoutSuite.providers.utils import run_concurrently -from ScoutSuite.core.console import print_error, print_debug class AWSFacade(AWSBaseFacade): @@ -29,6 +29,7 @@ def __init__(self, credentials: dict = None): self.efs = EFSFacade(self.session) self.elasticache = ElastiCacheFacade(self.session) self.emr = EMRFacade(self.session) + self.elbv2 = ELBv2Facade(self.session) async def build_region_list(self, service: str, chosen_regions=None, partition_name='aws'): service = 'ec2containerservice' if service == 'ecs' else service @@ -67,3 +68,4 @@ def _set_session(self, credentials: dict): self.efs = EFSFacade(self.session) self.elasticache = ElastiCacheFacade(self.session) self.emr = EMRFacade(self.session) + self.elbv2 = ELBv2Facade(self.session) From f441039419b731f0c0e1b689b87c6baaaf3464cb Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 16 Mar 2019 22:57:35 -0400 Subject: [PATCH 424/667] Define LoadBalancers resources. --- .../aws/resources/elbv2/load_balancers.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 ScoutSuite/providers/aws/resources/elbv2/load_balancers.py diff --git a/ScoutSuite/providers/aws/resources/elbv2/load_balancers.py b/ScoutSuite/providers/aws/resources/elbv2/load_balancers.py new file mode 100644 index 000000000..e67ca7f5a --- /dev/null +++ b/ScoutSuite/providers/aws/resources/elbv2/load_balancers.py @@ -0,0 +1,34 @@ +from ScoutSuite.providers.aws.resources.resources import AWSCompositeResources +from ScoutSuite.providers.utils import get_non_provider_id + +from .listeners import Listeners + + +class LoadBalancers(AWSCompositeResources): + _children = [ + (Listeners, 'listeners') + ] + + async def fetch_all(self, **kwargs): + raw_loads_balancers = await self.facade.elbv2.get_load_balancers(self.scope['region'], self.scope['vpc']) + # TODO: parallelize the following loop which is async: + for raw_load_balancer in raw_loads_balancers: + id, load_balancer = await self._parse_load_balancer(raw_load_balancer) + self[id] = load_balancer + await self._fetch_children( + parent=load_balancer, scope={'region': self.scope['region'], 'load_balancer_arn': load_balancer['arn']}) + + async def _parse_load_balancer(self, load_balancer): + load_balancer['arn'] = load_balancer.pop('LoadBalancerArn') + load_balancer['name'] = load_balancer.pop('LoadBalancerName') + load_balancer['security_groups'] = [] + + if 'SecurityGroups' in load_balancer: + for sg in load_balancer['SecurityGroups']: + load_balancer['security_groups'].append({'GroupId': sg}) + load_balancer.pop('SecurityGroups') + + load_balancer['attributes'] =\ + await self.facade.elbv2.get_load_balancer_attributes(self.scope['region'], load_balancer['arn']) + + return get_non_provider_id(load_balancer['name']), load_balancer From 839a00a77f785ffe4186864003304eb773959bd2 Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 16 Mar 2019 22:57:56 -0400 Subject: [PATCH 425/667] Define Listeners resources. --- .../providers/aws/resources/elbv2/listeners.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 ScoutSuite/providers/aws/resources/elbv2/listeners.py diff --git a/ScoutSuite/providers/aws/resources/elbv2/listeners.py b/ScoutSuite/providers/aws/resources/elbv2/listeners.py new file mode 100644 index 000000000..ce8c31e4a --- /dev/null +++ b/ScoutSuite/providers/aws/resources/elbv2/listeners.py @@ -0,0 +1,15 @@ +from ScoutSuite.providers.aws.resources.resources import AWSResources + + +class Listeners(AWSResources): + async def fetch_all(self, **kwargs): + listeners = await self.facade.elbv2.get_listeners(self.scope['region'], self.scope['load_balancer_arn']) + for raw_listener in listeners: + id, listener = self._parse_listener(raw_listener) + self[id] = listener + + def _parse_listener(self, raw_listener): + raw_listener.pop('ListenerArn') + raw_listener.pop('LoadBalancerArn') + port = raw_listener.pop('Port') + return port, raw_listener From 7b39218665216560639354508a7a8251efa400c4 Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 16 Mar 2019 22:58:13 -0400 Subject: [PATCH 426/667] Define ELBv2Vpcs resources. --- ScoutSuite/providers/aws/resources/elbv2/vpcs.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 ScoutSuite/providers/aws/resources/elbv2/vpcs.py diff --git a/ScoutSuite/providers/aws/resources/elbv2/vpcs.py b/ScoutSuite/providers/aws/resources/elbv2/vpcs.py new file mode 100644 index 000000000..4ec187715 --- /dev/null +++ b/ScoutSuite/providers/aws/resources/elbv2/vpcs.py @@ -0,0 +1,11 @@ +from ScoutSuite.providers.aws.resources.vpcs import Vpcs +from .load_balancers import LoadBalancers + + +class ELBv2Vpcs(Vpcs): + _children = [ + (LoadBalancers, 'lbs'), + ] + + def __init__(self, facade, scope: dict): + super(ELBv2Vpcs, self).__init__(facade, scope, add_ec2_classic=True) From ee5993e04681126b46fb8a46e21f7fcac4b9cd9a Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 16 Mar 2019 22:58:37 -0400 Subject: [PATCH 427/667] Define ELBv2 service. --- ScoutSuite/providers/aws/resources/elbv2/service.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 ScoutSuite/providers/aws/resources/elbv2/service.py diff --git a/ScoutSuite/providers/aws/resources/elbv2/service.py b/ScoutSuite/providers/aws/resources/elbv2/service.py new file mode 100644 index 000000000..6be76ee48 --- /dev/null +++ b/ScoutSuite/providers/aws/resources/elbv2/service.py @@ -0,0 +1,12 @@ +from ScoutSuite.providers.aws.resources.regions import Regions + +from .vpcs import ELBv2Vpcs + + +class ELBv2(Regions): + _children = [ + (ELBv2Vpcs, 'vpcs') + ] + + def __init__(self): + super(ELBv2, self).__init__('elbv2') From e86b3619ade234788ee27b40be47ccab8bfab8e9 Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 16 Mar 2019 22:59:36 -0400 Subject: [PATCH 428/667] Move helper. --- ScoutSuite/providers/aws/provider.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index e3da8c973..5e0ded86b 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -7,7 +7,6 @@ from ScoutSuite.core.console import print_debug, print_error, print_exception, print_info from ScoutSuite.providers.aws.configs.services import AWSServicesConfig from ScoutSuite.providers.aws.services.vpc import put_cidr_name -from ScoutSuite.providers.aws.services.elbv2 import check_security_group_rules from ScoutSuite.providers.base.configs.browser import combine_paths, get_object_at, get_value_at from ScoutSuite.providers.base.provider import BaseProvider from ScoutSuite.utils import manage_dictionary @@ -112,6 +111,18 @@ def _add_security_group_name_to_ec2_grants(self): {'AWSAccountId': self.aws_account_id}) def _add_security_group_data_to_elbv2(self): + def check_security_group_rules(lb, index, traffic_type): + none = 'N/A' + if traffic_type == 'ingress': + output = 'valid_inbound_rules' + elif traffic_type == 'egress': + output = 'valid_outbound_rules' + for protocol in lb['security_groups'][index]['rules'][traffic_type]['protocols']: + for port in lb['security_groups'][index]['rules'][traffic_type]['protocols'][protocol]['ports']: + lb['security_groups'][index][output] = True + if port not in lb['listeners'] and port != none: + lb['security_groups'][index][output] = False + ec2_config = self.services['ec2'] elbv2_config = self.services['elbv2'] for region in elbv2_config['regions']: From 16acb368492c72d5c5279bee79dc0c39c27d2206 Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 16 Mar 2019 23:00:28 -0400 Subject: [PATCH 429/667] Migrate ELBv2 service to the new architecture. --- ScoutSuite/providers/aws/configs/services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index 2c5311aef..1d637a951 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -9,7 +9,7 @@ from ScoutSuite.providers.aws.resources.efs.service import EFS from ScoutSuite.providers.aws.resources.elasticache.service import ElastiCache from ScoutSuite.providers.aws.services.elb import ELBConfig -from ScoutSuite.providers.aws.services.elbv2 import ELBv2Config +from ScoutSuite.providers.aws.resources.elbv2.service import ELBv2 from ScoutSuite.providers.aws.resources.emr.service import EMR from ScoutSuite.providers.aws.services.iam import IAMConfig from ScoutSuite.providers.aws.services.rds import RDSConfig @@ -62,7 +62,7 @@ def __init__(self, metadata=None, thread_config=4, **kwargs): self.efs = EFS() self.elasticache = ElastiCache() self.elb = ELBConfig(metadata['compute']['elb'], thread_config) - self.elbv2 = ELBv2Config(metadata['compute']['elbv2'], thread_config) + self.elbv2 = ELBv2() self.emr = EMR() self.iam = IAMConfig(thread_config) self.awslambda = Lambdas() From c78223c2da273d53cb17fda5daf0814ac93c7c64 Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 16 Mar 2019 23:01:21 -0400 Subject: [PATCH 430/667] Remove old ELBv2 service support. --- ScoutSuite/providers/aws/services/elbv2.py | 92 ---------------------- 1 file changed, 92 deletions(-) delete mode 100644 ScoutSuite/providers/aws/services/elbv2.py diff --git a/ScoutSuite/providers/aws/services/elbv2.py b/ScoutSuite/providers/aws/services/elbv2.py deleted file mode 100644 index aecd31b89..000000000 --- a/ScoutSuite/providers/aws/services/elbv2.py +++ /dev/null @@ -1,92 +0,0 @@ -# -*- coding: utf-8 -*- -""" -ELBv2-related classes and functions -""" - -from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients -from ScoutSuite.providers.aws.configs.vpc import VPCConfig -from ScoutSuite.utils import manage_dictionary -from ScoutSuite.providers.aws.utils import ec2_classic, handle_truncated_response - - -######################################## -# ELBv2RegionConfig -######################################## - -# noinspection PyBroadException -class ELBv2RegionConfig(RegionConfig): - """ - ELBv2 configuration for a single AWS region - - :ivar vpcs: Dictionary of VPCs [id] - """ - ssl_policies = {} - - def parse_lb(self, global_params, region, lb): - """ - - :param lb: - :param global_params: - :param region: Name of the AWS region - :return: - """ - lb['arn'] = lb.pop('LoadBalancerArn') - lb['name'] = lb.pop('LoadBalancerName') - vpc_id = lb.pop('VpcId') if 'VpcId' in lb and lb['VpcId'] else ec2_classic - manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types)) - lb['security_groups'] = [] - try: - for sg in lb['SecurityGroups']: - lb['security_groups'].append({'GroupId': sg}) - lb.pop('SecurityGroups') - except Exception as e: - # Network load balancers do not have security groups - pass - lb['has_secure_protocol'] = False - lb['listeners'] = {} - # Get listeners - listeners = handle_truncated_response(api_clients[region].describe_listeners, {'LoadBalancerArn': lb['arn']}, - ['Listeners'])['Listeners'] - for listener in listeners: - listener.pop('ListenerArn') - listener.pop('LoadBalancerArn') - port = listener.pop('Port') - lb['listeners'][port] = listener - if listener['Protocol'] is 'HTTPS': - lb['has_secure_protocol'] = True - - # Get attributes - lb['attributes'] = api_clients[region].describe_load_balancer_attributes(LoadBalancerArn=lb['arn'])[ - 'Attributes'] - self.vpcs[vpc_id].lbs[self.get_non_provider_id(lb['name'])] = lb - - def parse_ssl_policie(self, global_params, region, policy): - id = self.get_non_provider_id(policy['Name']) - self.ssl_policies[id] = policy - - -######################################## -# ELBv2Config -######################################## - -class ELBv2Config(RegionalServiceConfig): - """ - ELBv2 configuration for all AWS regions - """ - region_config_class = ELBv2RegionConfig - - def __init__(self, service_metadata, thread_config=4): - super(ELBv2Config, self).__init__(service_metadata, thread_config) - - -def check_security_group_rules(lb, index, traffic_type): - none = 'N/A' - if traffic_type == 'ingress': - output = 'valid_inbound_rules' - elif traffic_type == 'egress': - output = 'valid_outbound_rules' - for protocol in lb['security_groups'][index]['rules'][traffic_type]['protocols']: - for port in lb['security_groups'][index]['rules'][traffic_type]['protocols'][protocol]['ports']: - lb['security_groups'][index][output] = True - if port not in lb['listeners'] and port != none: - lb['security_groups'][index][output] = False From f6f9bb86cc44baa4c7151c45b016ae1a548f25c1 Mon Sep 17 00:00:00 2001 From: misg Date: Sun, 17 Mar 2019 19:44:39 -0400 Subject: [PATCH 431/667] Define a facade for Redshift service. --- ScoutSuite/providers/aws/facade/redshift.py | 45 +++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 ScoutSuite/providers/aws/facade/redshift.py diff --git a/ScoutSuite/providers/aws/facade/redshift.py b/ScoutSuite/providers/aws/facade/redshift.py new file mode 100644 index 000000000..05f4d42de --- /dev/null +++ b/ScoutSuite/providers/aws/facade/redshift.py @@ -0,0 +1,45 @@ +from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils +from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade +from ScoutSuite.providers.aws.utils import ec2_classic +from botocore.utils import ClientError + +from asyncio import Lock + + +class RedshiftFacade(AWSBaseFacade): + regional_cluster_cache_locks = {} + clusters_cache = {} + + async def get_clusters(self, region: str, vpc: str): + await self.cache_clusters(region) + return [cluster for cluster in self.clusters_cache[region] if cluster['VpcId'] == vpc] + + async def cache_clusters(self, region): + async with self.regional_cluster_cache_locks.setdefault(region, Lock()): + if region in self.clusters_cache: + return + + self.clusters_cache[region] =\ + await AWSFacadeUtils.get_all_pages('redshift', region, self.session, + 'describe_clusters', 'Clusters') + + for cluster in self.clusters_cache[region]: + cluster['VpcId'] =\ + cluster['VpcId'] if 'VpcId' in cluster and cluster['VpcId'] else ec2_classic + + async def get_cluster_parameter_groups(self, region: str): + return await AWSFacadeUtils.get_all_pages( + 'redshift', region, self.session, 'describe_cluster_parameter_groups', 'ParameterGroups') + + async def get_cluster_security_groups(self, region: str): + # For VPC-by-default customers, describe_cluster_parameters will throw an exception. Just try and ignore it: + try: + return await AWSFacadeUtils.get_all_pages( + 'redshift', region, self.session, 'describe_cluster_security_groups', 'ClusterSecurityGroups') + except ClientError: + return [] + + async def get_cluster_parameters(self, region: str, parameter_group: str): + return await AWSFacadeUtils.get_all_pages( + 'redshift', region, self.session, 'describe_cluster_parameters', 'Parameters', + ParameterGroupName=parameter_group) From b843210acb90ecf4f60de2e93f9febdd7d08bc51 Mon Sep 17 00:00:00 2001 From: misg Date: Sun, 17 Mar 2019 19:45:02 -0400 Subject: [PATCH 432/667] Add Redshift facade to AWS facade. --- ScoutSuite/providers/aws/facade/facade.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/facade/facade.py b/ScoutSuite/providers/aws/facade/facade.py index 92d09fa63..e14b73080 100644 --- a/ScoutSuite/providers/aws/facade/facade.py +++ b/ScoutSuite/providers/aws/facade/facade.py @@ -11,9 +11,9 @@ from ScoutSuite.providers.aws.facade.efs import EFSFacade from ScoutSuite.providers.aws.facade.elasticache import ElastiCacheFacade from ScoutSuite.providers.aws.facade.emr import EMRFacade +from ScoutSuite.providers.aws.facade.redshift import RedshiftFacade from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade from ScoutSuite.providers.utils import run_concurrently -from ScoutSuite.core.console import print_error, print_debug class AWSFacade(AWSBaseFacade): @@ -29,6 +29,7 @@ def __init__(self, credentials: dict = None): self.efs = EFSFacade(self.session) self.elasticache = ElastiCacheFacade(self.session) self.emr = EMRFacade(self.session) + self.redshift = RedshiftFacade(self.session) async def build_region_list(self, service: str, chosen_regions=None, partition_name='aws'): service = 'ec2containerservice' if service == 'ecs' else service @@ -67,3 +68,4 @@ def _set_session(self, credentials: dict): self.efs = EFSFacade(self.session) self.elasticache = ElastiCacheFacade(self.session) self.emr = EMRFacade(self.session) + self.redshift = RedshiftFacade(self.session) From 581120c732856bcea7853a39c2d35faf33cf5e5e Mon Sep 17 00:00:00 2001 From: misg Date: Sun, 17 Mar 2019 19:45:18 -0400 Subject: [PATCH 433/667] Define Clusters resources. --- .../providers/aws/resources/redshift/clusters.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 ScoutSuite/providers/aws/resources/redshift/clusters.py diff --git a/ScoutSuite/providers/aws/resources/redshift/clusters.py b/ScoutSuite/providers/aws/resources/redshift/clusters.py new file mode 100644 index 000000000..ea65e6aa2 --- /dev/null +++ b/ScoutSuite/providers/aws/resources/redshift/clusters.py @@ -0,0 +1,15 @@ +from ScoutSuite.providers.aws.resources.resources import AWSResources + + +class Clusters(AWSResources): + async def fetch_all(self, **kwargs): + raw_clusters = await self.facade.redshift.get_clusters(self.scope['region'], self.scope['vpc']) + for raw_cluster in raw_clusters: + id, cluster = self._parse_cluster(raw_cluster) + self[id] = cluster + + def _parse_cluster(self, raw_cluster): + name = raw_cluster.pop('ClusterIdentifier') + raw_cluster['name'] = name + + return name, raw_cluster From db7068b00d18803ab3ee6e81b1a49bd1a6dc59f3 Mon Sep 17 00:00:00 2001 From: misg Date: Sun, 17 Mar 2019 19:45:38 -0400 Subject: [PATCH 434/667] Define ClusterSecurityGroups resources. --- .../resources/redshift/cluster_security_groups.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 ScoutSuite/providers/aws/resources/redshift/cluster_security_groups.py diff --git a/ScoutSuite/providers/aws/resources/redshift/cluster_security_groups.py b/ScoutSuite/providers/aws/resources/redshift/cluster_security_groups.py new file mode 100644 index 000000000..ea91352ba --- /dev/null +++ b/ScoutSuite/providers/aws/resources/redshift/cluster_security_groups.py @@ -0,0 +1,14 @@ +from ScoutSuite.providers.aws.resources.resources import AWSResources + + +class ClusterSecurityGroups(AWSResources): + async def fetch_all(self, **kwargs): + raw_security_groups = await self.facade.redshift.get_cluster_security_groups(self.scope['region']) + for raw_security_group in raw_security_groups: + id, security_group = self._parse_security_group(raw_security_group) + self[id] = security_group + + def _parse_security_group(self, raw_security_group): + name = raw_security_group.pop('ClusterSecurityGroupName') + raw_security_group['name'] = name + return name, raw_security_group From 899edd8b0f16d58c407c7d7818d09684c4a8c0b4 Mon Sep 17 00:00:00 2001 From: misg Date: Sun, 17 Mar 2019 19:46:03 -0400 Subject: [PATCH 435/667] Define ClusterParameterGroups resources. --- .../redshift/cluster_parameter_groups.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 ScoutSuite/providers/aws/resources/redshift/cluster_parameter_groups.py diff --git a/ScoutSuite/providers/aws/resources/redshift/cluster_parameter_groups.py b/ScoutSuite/providers/aws/resources/redshift/cluster_parameter_groups.py new file mode 100644 index 000000000..cd0240594 --- /dev/null +++ b/ScoutSuite/providers/aws/resources/redshift/cluster_parameter_groups.py @@ -0,0 +1,28 @@ +from ScoutSuite.providers.aws.resources.resources import AWSCompositeResources +from ScoutSuite.providers.utils import get_non_provider_id + +from .cluster_parameters import ClusterParameters + + +class ClusterParameterGroups(AWSCompositeResources): + _children = [ + (ClusterParameters, 'parameters') + ] + + async def fetch_all(self, **kwargs): + raw_parameter_groups = await self.facade.redshift.get_cluster_parameter_groups(self.scope['region']) + # TODO: parallelize this async loop: + for raw_parameter_group in raw_parameter_groups: + id, parameter_group = self._parse_parameter_group(raw_parameter_group) + await self._fetch_children( + parent=parameter_group, + scope={'region': self.scope['region'], 'parameter_group_name': parameter_group['name']} + ) + self[id] = parameter_group + + def _parse_parameter_group(self, raw_parameter_group): + name = raw_parameter_group.pop('ParameterGroupName') + id = get_non_provider_id(name) + raw_parameter_group['name'] = name + + return id, raw_parameter_group From d9e85a85ef3397dfd08af01b2fdd3177af556a96 Mon Sep 17 00:00:00 2001 From: misg Date: Sun, 17 Mar 2019 19:46:20 -0400 Subject: [PATCH 436/667] Define ClusterParameters resources. --- .../aws/resources/redshift/cluster_parameters.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 ScoutSuite/providers/aws/resources/redshift/cluster_parameters.py diff --git a/ScoutSuite/providers/aws/resources/redshift/cluster_parameters.py b/ScoutSuite/providers/aws/resources/redshift/cluster_parameters.py new file mode 100644 index 000000000..32aa25b76 --- /dev/null +++ b/ScoutSuite/providers/aws/resources/redshift/cluster_parameters.py @@ -0,0 +1,15 @@ +from ScoutSuite.providers.aws.resources.resources import AWSResources + + +class ClusterParameters(AWSResources): + async def fetch_all(self, **kwargs): + raw_parameters = await self.facade.redshift.get_cluster_parameters( + self.scope['region'], self.scope['parameter_group_name']) + for raw_parameter in raw_parameters: + id, parameter = self._parse_parameter(raw_parameter) + self[id] = parameter + + def _parse_parameter(self, raw_parameter): + parameter = {'value': raw_parameter['ParameterValue'], + 'source': raw_parameter['Source']} + return raw_parameter['ParameterName'], parameter From 21b4240cb620a7b4be8786e535ae920bace12a50 Mon Sep 17 00:00:00 2001 From: misg Date: Sun, 17 Mar 2019 19:47:40 -0400 Subject: [PATCH 437/667] Define RedshiftVpcs resources. --- ScoutSuite/providers/aws/resources/redshift/vpcs.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 ScoutSuite/providers/aws/resources/redshift/vpcs.py diff --git a/ScoutSuite/providers/aws/resources/redshift/vpcs.py b/ScoutSuite/providers/aws/resources/redshift/vpcs.py new file mode 100644 index 000000000..3d218bc91 --- /dev/null +++ b/ScoutSuite/providers/aws/resources/redshift/vpcs.py @@ -0,0 +1,12 @@ +from ScoutSuite.providers.aws.resources.vpcs import Vpcs + +from .clusters import Clusters + + +class RedshiftVpcs(Vpcs): + _children = [ + (Clusters, 'clusters'), + ] + + def __init__(self, facade, scope: dict): + super(RedshiftVpcs, self).__init__(facade, scope, add_ec2_classic=True) From ff7a26e3c0ee43cf09e84a30f622198daa2c7fc7 Mon Sep 17 00:00:00 2001 From: misg Date: Sun, 17 Mar 2019 19:47:48 -0400 Subject: [PATCH 438/667] Define Redshift service. --- .../providers/aws/resources/redshift/service.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 ScoutSuite/providers/aws/resources/redshift/service.py diff --git a/ScoutSuite/providers/aws/resources/redshift/service.py b/ScoutSuite/providers/aws/resources/redshift/service.py new file mode 100644 index 000000000..d8b24ae4c --- /dev/null +++ b/ScoutSuite/providers/aws/resources/redshift/service.py @@ -0,0 +1,16 @@ +from ScoutSuite.providers.aws.resources.regions import Regions + +from .vpcs import RedshiftVpcs +from .cluster_parameter_groups import ClusterParameterGroups +from .cluster_security_groups import ClusterSecurityGroups + + +class Redshift(Regions): + _children = [ + (RedshiftVpcs, 'vpcs'), + (ClusterParameterGroups, 'parameter_groups'), + (ClusterSecurityGroups, 'security_groups') + ] + + def __init__(self): + super(Redshift, self).__init__('redshift') From 6527f81e0612d24efead95a66eae5d9b11ef1a38 Mon Sep 17 00:00:00 2001 From: misg Date: Sun, 17 Mar 2019 19:48:22 -0400 Subject: [PATCH 439/667] Migrate Redshift service to the new architecture. --- ScoutSuite/providers/aws/configs/services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index 2c5311aef..a46693d97 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -13,7 +13,7 @@ from ScoutSuite.providers.aws.resources.emr.service import EMR from ScoutSuite.providers.aws.services.iam import IAMConfig from ScoutSuite.providers.aws.services.rds import RDSConfig -from ScoutSuite.providers.aws.services.redshift import RedshiftConfig +from ScoutSuite.providers.aws.resources.redshift.service import Redshift from ScoutSuite.providers.aws.services.route53 import Route53Config, Route53DomainsConfig from ScoutSuite.providers.aws.services.s3 import S3Config from ScoutSuite.providers.aws.services.ses import SESConfig @@ -66,7 +66,7 @@ def __init__(self, metadata=None, thread_config=4, **kwargs): self.emr = EMR() self.iam = IAMConfig(thread_config) self.awslambda = Lambdas() - self.redshift = RedshiftConfig(metadata['database']['redshift'], thread_config) + self.redshift = Redshift() self.rds = RDSConfig(metadata['database']['rds'], thread_config) self.route53 = Route53Config(thread_config) self.route53domains = Route53DomainsConfig(thread_config) From 82a2c52e7770a249494cfd7edcc85bb07370bbe6 Mon Sep 17 00:00:00 2001 From: misg Date: Sun, 17 Mar 2019 19:48:47 -0400 Subject: [PATCH 440/667] Remove old Redshift service support. --- ScoutSuite/providers/aws/metadata.json | 7 -- ScoutSuite/providers/aws/services/redshift.py | 81 ------------------- 2 files changed, 88 deletions(-) delete mode 100644 ScoutSuite/providers/aws/services/redshift.py diff --git a/ScoutSuite/providers/aws/metadata.json b/ScoutSuite/providers/aws/metadata.json index 0b4c2b7bd..d9f6cc7ef 100644 --- a/ScoutSuite/providers/aws/metadata.json +++ b/ScoutSuite/providers/aws/metadata.json @@ -481,8 +481,6 @@ "redshift": { "resources": { "clusters": { - "api_call": "describe_clusters", - "response": "Clusters", "cols": 2, "path": "services.redshift.regions.id.vpcs.id.clusters", "callbacks": [ @@ -491,15 +489,10 @@ ] }, "parameter_groups": { - "api_call": "describe_cluster_parameter_groups", - "response": "ParameterGroups", "cols": 2, "path": "services.redshift.regions.id.parameter_groups" }, "security_groups": { - "api_call": "describe_cluster_security_groups", - "response": "ClusterSecurityGroups", - "no_exceptions": true, "cols": 2, "path": "services.redshift.regions.id.security_groups" } diff --git a/ScoutSuite/providers/aws/services/redshift.py b/ScoutSuite/providers/aws/services/redshift.py deleted file mode 100644 index e51c6d503..000000000 --- a/ScoutSuite/providers/aws/services/redshift.py +++ /dev/null @@ -1,81 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Redshift-related classes and functions -""" - -from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients -from ScoutSuite.providers.aws.configs.vpc import VPCConfig -from ScoutSuite.utils import manage_dictionary -from ScoutSuite.providers.aws.utils import ec2_classic, handle_truncated_response - - -######################################## -# RedshiftRegionConfig -######################################## - -class RedshiftRegionConfig(RegionConfig): - """ - Redshift configuration for a single AWS region - """ - parameter_groups = {} - - def parse_cluster(self, global_params, region, cluster): - """ - Parse a single Redshift cluster - - :param global_params: Parameters shared for all regions - :param region: Name of the AWS region - :param cluster: Cluster - """ - vpc_id = cluster.pop('VpcId') if 'VpcId' in cluster else ec2_classic - manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types)) - name = cluster.pop('ClusterIdentifier') - cluster['name'] = name - self.vpcs[vpc_id].clusters[name] = cluster - - def parse_parameter_group(self, global_params, region, parameter_group): - """ - Parse a single Redshift parameter group and fetch all of its parameters - - :param global_params: Parameters shared for all regions - :param region: Name of the AWS region - :param parameter_group: Parameter group - """ - pg_name = parameter_group.pop('ParameterGroupName') - pg_id = self.get_non_provider_id(pg_name) # Name could be used as only letters digits or hyphens - parameter_group['name'] = pg_name - parameter_group['parameters'] = {} - api_client = api_clients[region] - parameters = handle_truncated_response(api_client.describe_cluster_parameters, {'ParameterGroupName': pg_name}, - ['Parameters'])['Parameters'] - for parameter in parameters: - param = {'value': parameter['ParameterValue'], 'source': parameter['Source']} - parameter_group['parameters'][parameter['ParameterName']] = param - self.parameter_groups[pg_id] = parameter_group - - def parse_security_group(self, global_params, region, security_group): - """ - Parse a single Redsfhit security group - - :param security_group: - :param global_params: Parameters shared for all regions - :param region: Name of the AWS region - """ - name = security_group.pop('ClusterSecurityGroupName') - security_group['name'] = name - self.security_groups['name'] = security_group - - -######################################## -# RedshiftConfig -######################################## - -class RedshiftConfig(RegionalServiceConfig): - """ - Redshift configuration for all AWS regions - """ - - region_config_class = RedshiftRegionConfig - - def __init__(self, service_metadata, thread_config=4): - super(RedshiftConfig, self).__init__(service_metadata, thread_config) From 77fd9efd0fa4a8adceb71c1fdcbc2ba762b9a900 Mon Sep 17 00:00:00 2001 From: misg Date: Sun, 17 Mar 2019 19:51:28 -0400 Subject: [PATCH 441/667] Update metadata.json --- ScoutSuite/providers/aws/metadata.json | 6 ------ 1 file changed, 6 deletions(-) diff --git a/ScoutSuite/providers/aws/metadata.json b/ScoutSuite/providers/aws/metadata.json index 0b4c2b7bd..caf14354d 100644 --- a/ScoutSuite/providers/aws/metadata.json +++ b/ScoutSuite/providers/aws/metadata.json @@ -278,18 +278,12 @@ "elb": { "resources": { "elbs": { - "api_call": "describe_load_balancers", - "response": "LoadBalancerDescriptions", "cols": 2, "path": "services.elb.regions.id.vpcs.id.elbs", "callbacks": [ [ "match_security_groups_and_resources_callback", {"status_path": ["Scheme"], "sg_list_attribute_name": ["security_groups"], "sg_id_attribute_name": "GroupId"} ], [ "get_lb_attack_surface", {} ] ] - }, - "elb_policies": { - "cols": 2, - "path": "services.elb.regions.id.elb_policies" } }, "summaries": { From c616e4ec7821f14d7d406ab89320be944541ae60 Mon Sep 17 00:00:00 2001 From: misg Date: Sun, 17 Mar 2019 19:56:26 -0400 Subject: [PATCH 442/667] Update metadata.json --- ScoutSuite/providers/aws/metadata.json | 8 -------- 1 file changed, 8 deletions(-) diff --git a/ScoutSuite/providers/aws/metadata.json b/ScoutSuite/providers/aws/metadata.json index 0b4c2b7bd..024d08afa 100644 --- a/ScoutSuite/providers/aws/metadata.json +++ b/ScoutSuite/providers/aws/metadata.json @@ -302,20 +302,12 @@ "elbv2": { "resources": { "lbs": { - "api_call": "describe_load_balancers", - "response": "LoadBalancers", "cols": 2, "path": "services.elbv2.regions.id.vpcs.id.lbs", "callbacks": [ [ "match_security_groups_and_resources_callback", {"status_path": ["State", "Code"], "sg_list_attribute_name": ["security_groups"], "sg_id_attribute_name": "GroupId"} ], [ "get_lb_attack_surface", {} ] ] - }, - "ssl_policies": { - "api_call": "describe_ssl_policies", - "response": "SslPolicies", - "hidden": true, - "path": "services.elbv2.ssl_policies" } }, "summaries": { From 566050376b731602d5c5a5a563fc952f8e9237d4 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 18 Mar 2019 00:22:43 -0400 Subject: [PATCH 443/667] Define a facade for SQS service. --- ScoutSuite/providers/aws/facade/sqs.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 ScoutSuite/providers/aws/facade/sqs.py diff --git a/ScoutSuite/providers/aws/facade/sqs.py b/ScoutSuite/providers/aws/facade/sqs.py new file mode 100644 index 000000000..9795d1d9b --- /dev/null +++ b/ScoutSuite/providers/aws/facade/sqs.py @@ -0,0 +1,20 @@ +from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade +from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils +from ScoutSuite.providers.utils import run_concurrently + + +class SQSFacade(AWSBaseFacade): + async def get_queues(self, region: str): + sqs_client = AWSFacadeUtils.get_client('sqs', region, self.session) + queues = await run_concurrently(sqs_client.list_queues) + + if 'QueueUrls' in queues: + return queues['QueueUrls'] + + return [] + + async def get_queue_attributes(self, region: str, queue_url: str, attribute_names: []): + sqs_client = AWSFacadeUtils.get_client('sqs', region, self.session) + return await run_concurrently( + lambda: sqs_client.get_queue_attributes(QueueUrl=queue_url, AttributeNames=attribute_names)['Attributes'] + ) From 54ca97f39e52f0c1c11c66bc1e67be8930652906 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 18 Mar 2019 00:22:59 -0400 Subject: [PATCH 444/667] Add SQS facade to AWS facade. --- ScoutSuite/providers/aws/facade/facade.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/facade/facade.py b/ScoutSuite/providers/aws/facade/facade.py index 92d09fa63..69929dec4 100644 --- a/ScoutSuite/providers/aws/facade/facade.py +++ b/ScoutSuite/providers/aws/facade/facade.py @@ -11,9 +11,9 @@ from ScoutSuite.providers.aws.facade.efs import EFSFacade from ScoutSuite.providers.aws.facade.elasticache import ElastiCacheFacade from ScoutSuite.providers.aws.facade.emr import EMRFacade +from ScoutSuite.providers.aws.facade.sqs import SQSFacade from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade from ScoutSuite.providers.utils import run_concurrently -from ScoutSuite.core.console import print_error, print_debug class AWSFacade(AWSBaseFacade): @@ -29,6 +29,7 @@ def __init__(self, credentials: dict = None): self.efs = EFSFacade(self.session) self.elasticache = ElastiCacheFacade(self.session) self.emr = EMRFacade(self.session) + self.sqs = SQSFacade(self.session) async def build_region_list(self, service: str, chosen_regions=None, partition_name='aws'): service = 'ec2containerservice' if service == 'ecs' else service @@ -67,3 +68,4 @@ def _set_session(self, credentials: dict): self.efs = EFSFacade(self.session) self.elasticache = ElastiCacheFacade(self.session) self.emr = EMRFacade(self.session) + self.sqs = SQSFacade(self.session) From 50d0a1e3cd7d13a392a4e9018a9818fc8eb5ee8d Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 18 Mar 2019 00:23:20 -0400 Subject: [PATCH 445/667] Define RegionalQueues and SQS resources. --- .../providers/aws/resources/sqs/service.py | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 ScoutSuite/providers/aws/resources/sqs/service.py diff --git a/ScoutSuite/providers/aws/resources/sqs/service.py b/ScoutSuite/providers/aws/resources/sqs/service.py new file mode 100644 index 000000000..dfe34ed79 --- /dev/null +++ b/ScoutSuite/providers/aws/resources/sqs/service.py @@ -0,0 +1,39 @@ +import json + +from ScoutSuite.providers.aws.resources.regions import Regions +from ScoutSuite.providers.aws.resources.resources import AWSResources + + +class RegionalQueues(AWSResources): + async def fetch_all(self, **kwargs): + queue_urls = await self.facade.sqs.get_queues(self.scope['region']) + # TODO: parallelize this async loop: + for queue_url in queue_urls: + id, queue = await self._parse_queue(queue_url) + self[id] = queue + + async def _parse_queue(self, queue_url): + queue = {'QueueUrl': queue_url} + attributes = await self.facade.sqs.get_queue_attributes( + self.scope['region'], queue_url, ['CreatedTimestamp', 'Policy', 'QueueArn', 'KmsMasterKeyId'] + ) + queue['arn'] = attributes.pop('QueueArn') + queue['name'] = queue['arn'].split(':')[-1] + queue['kms_master_key_id'] = attributes.pop('KmsMasterKeyId', None) + queue['CreatedTimestamp'] = attributes.pop('CreatedTimestamp', None) + + if 'Policy' in attributes: + queue['Policy'] = json.loads(attributes['Policy']) + else: + queue['Policy'] = {'Statement': []} + + return queue['name'], queue + + +class SQS(Regions): + _children = [ + (RegionalQueues, 'queues') + ] + + def __init__(self): + super(SQS, self).__init__('sqs') From b88725952291ee9bbeed56dd1da6e34dd6cd0f86 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 18 Mar 2019 00:23:40 -0400 Subject: [PATCH 446/667] Migrate SQS service to the new architecture. --- ScoutSuite/providers/aws/configs/services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index 2c5311aef..624932d9c 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -18,7 +18,7 @@ from ScoutSuite.providers.aws.services.s3 import S3Config from ScoutSuite.providers.aws.services.ses import SESConfig from ScoutSuite.providers.aws.services.sns import SNSConfig -from ScoutSuite.providers.aws.services.sqs import SQSConfig +from ScoutSuite.providers.aws.resources.sqs.service import SQS from ScoutSuite.providers.aws.services.vpc import VPCConfig from ScoutSuite.providers.base.configs.services import BaseServicesConfig @@ -73,7 +73,7 @@ def __init__(self, metadata=None, thread_config=4, **kwargs): self.s3 = S3Config(thread_config) self.ses = SESConfig(metadata['messaging']['ses'], thread_config) self.sns = SNSConfig(metadata['messaging']['sns'], thread_config) - self.sqs = SQSConfig(metadata['messaging']['sqs'], thread_config) + self.sqs = SQS() self.vpc = VPCConfig(metadata['network']['vpc'], thread_config) try: From 8c6689e3cdb52c4918c522a69a0ed0334ab252fb Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 18 Mar 2019 00:24:08 -0400 Subject: [PATCH 447/667] Update metadata.json. --- ScoutSuite/providers/aws/metadata.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/ScoutSuite/providers/aws/metadata.json b/ScoutSuite/providers/aws/metadata.json index 0b4c2b7bd..0e77fec19 100644 --- a/ScoutSuite/providers/aws/metadata.json +++ b/ScoutSuite/providers/aws/metadata.json @@ -103,8 +103,6 @@ "sqs" : { "resources": { "queues": { - "api_call": "list_queues", - "response": "QueueUrls", "cols": 2, "path": "services.sqs.regions.id.queues" } From a1c0df375cb06fc28a7e17d2f010d3f767cde5c5 Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Mon, 18 Mar 2019 10:37:11 +0100 Subject: [PATCH 448/667] Update README --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e71fe277b..f2a58fb36 100644 --- a/README.md +++ b/README.md @@ -30,8 +30,8 @@ The latest (and final) version of Scout2 can be found in --password + +Additional information can be found in [the wiki](https://github.com/nccgroup/ScoutSuite/wiki). \ No newline at end of file From 6434e1c2fcdfd2fcade2ac666e22e9a2c6e38fd6 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 18 Mar 2019 10:45:30 -0400 Subject: [PATCH 449/667] Fetch queue attributes just after listing SQS queues. --- ScoutSuite/providers/aws/facade/sqs.py | 28 ++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/sqs.py b/ScoutSuite/providers/aws/facade/sqs.py index 9795d1d9b..b50accd76 100644 --- a/ScoutSuite/providers/aws/facade/sqs.py +++ b/ScoutSuite/providers/aws/facade/sqs.py @@ -2,19 +2,35 @@ from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils from ScoutSuite.providers.utils import run_concurrently +import asyncio + class SQSFacade(AWSBaseFacade): - async def get_queues(self, region: str): + async def get_queues(self, region: str, attribute_names: []): sqs_client = AWSFacadeUtils.get_client('sqs', region, self.session) - queues = await run_concurrently(sqs_client.list_queues) + raw_queues = await run_concurrently(sqs_client.list_queues) + + if 'QueueUrls' not in raw_queues: + return [] - if 'QueueUrls' in queues: - return queues['QueueUrls'] + queue_urls = raw_queues['QueueUrls'] + # Fetch the attributes of all the queues concurrently:: + tasks = { + asyncio.ensure_future( + self.get_queue_attributes(region, queue_url, attribute_names) + ) for queue_url in queue_urls + } + queues = [] + for result in asyncio.as_completed(tasks): + queue_url, queue_attributes = await result + queues.append((queue_url, queue_attributes)) - return [] + return queues async def get_queue_attributes(self, region: str, queue_url: str, attribute_names: []): sqs_client = AWSFacadeUtils.get_client('sqs', region, self.session) - return await run_concurrently( + queue_attributes = await run_concurrently( lambda: sqs_client.get_queue_attributes(QueueUrl=queue_url, AttributeNames=attribute_names)['Attributes'] ) + + return queue_url, queue_attributes From ee4e1ed6ca9d7d990f003d7754970c450aac7409 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 18 Mar 2019 10:46:00 -0400 Subject: [PATCH 450/667] Make _parse_queue only a parse method. --- .../providers/aws/resources/sqs/service.py | 26 +++++++++---------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/sqs/service.py b/ScoutSuite/providers/aws/resources/sqs/service.py index dfe34ed79..254786aaf 100644 --- a/ScoutSuite/providers/aws/resources/sqs/service.py +++ b/ScoutSuite/providers/aws/resources/sqs/service.py @@ -6,24 +6,22 @@ class RegionalQueues(AWSResources): async def fetch_all(self, **kwargs): - queue_urls = await self.facade.sqs.get_queues(self.scope['region']) - # TODO: parallelize this async loop: - for queue_url in queue_urls: - id, queue = await self._parse_queue(queue_url) + queues = await self.facade.sqs.get_queues( + self.scope['region'], ['CreatedTimestamp', 'Policy', 'QueueArn', 'KmsMasterKeyId']) + for queue_url, queue_attributes in queues: + id, queue = self._parse_queue(queue_url, queue_attributes) self[id] = queue - async def _parse_queue(self, queue_url): - queue = {'QueueUrl': queue_url} - attributes = await self.facade.sqs.get_queue_attributes( - self.scope['region'], queue_url, ['CreatedTimestamp', 'Policy', 'QueueArn', 'KmsMasterKeyId'] - ) - queue['arn'] = attributes.pop('QueueArn') + def _parse_queue(self, queue_url, queue_attributes): + queue = {} + queue['QueueUrl'] = queue_url + queue['arn'] = queue_attributes.pop('QueueArn') queue['name'] = queue['arn'].split(':')[-1] - queue['kms_master_key_id'] = attributes.pop('KmsMasterKeyId', None) - queue['CreatedTimestamp'] = attributes.pop('CreatedTimestamp', None) + queue['kms_master_key_id'] = queue_attributes.pop('KmsMasterKeyId', None) + queue['CreatedTimestamp'] = queue_attributes.pop('CreatedTimestamp', None) - if 'Policy' in attributes: - queue['Policy'] = json.loads(attributes['Policy']) + if 'Policy' in queue_attributes: + queue['Policy'] = json.loads(queue_attributes['Policy']) else: queue['Policy'] = {'Statement': []} From e9e7a23e0d0150be47244fa7e49fb6e52c59ef93 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 18 Mar 2019 11:39:33 -0400 Subject: [PATCH 451/667] Added policies normalization --- ScoutSuite/providers/aws/facade/iam.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/facade/iam.py b/ScoutSuite/providers/aws/facade/iam.py index 2f84eb9b9..d3204e7a5 100644 --- a/ScoutSuite/providers/aws/facade/iam.py +++ b/ScoutSuite/providers/aws/facade/iam.py @@ -164,11 +164,25 @@ def _get_inline_policies(self, iam_resource_type, resource_id, resource_name): policy_document = get_policy_method(**args)['PolicyDocument'] policy_id = get_non_provider_id(policy_name) fetched_policies[policy_id] = {} - fetched_policies[policy_id]['PolicyDocument'] = policy_document + fetched_policies[policy_id]['PolicyDocument'] = self._normalize_statements(policy_document) fetched_policies[policy_id]['name'] = policy_name + fetched_policies[policy_id] = fetched_policies[policy_id] except Exception as e: if is_throttled(e): raise e else: print_exception(e) return fetched_policies + + def _normalize_statements(self, policy_document) : + for statement in policy_document['Statement']: + # Action or NotAction + action_string = 'Action' if 'Action' in statement else 'NotAction' + if type(statement[action_string]) != list: + statement[action_string] = [statement[action_string]] + # Resource or NotResource + resource_string = 'Resource' if 'Resource' in statement else 'NotResource' + if type(statement[resource_string]) != list: + statement[resource_string] = [statement[resource_string]] + + return policy_document \ No newline at end of file From 35eb74c40b40fb77d572c30536d055e1757e7501 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 18 Mar 2019 11:39:39 -0400 Subject: [PATCH 452/667] Added null check --- ScoutSuite/providers/aws/resources/iam/roles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/resources/iam/roles.py b/ScoutSuite/providers/aws/resources/iam/roles.py index 945e5067d..8a0893dc7 100644 --- a/ScoutSuite/providers/aws/resources/iam/roles.py +++ b/ScoutSuite/providers/aws/resources/iam/roles.py @@ -12,6 +12,6 @@ def _parse_role(self, raw_role): raw_role['id'] = raw_role.pop('RoleId') raw_role['name'] = raw_role.pop('RoleName') raw_role['arn'] = raw_role.pop('Arn') - raw_role.pop('Description') + if 'Description' in raw_role: raw_role.pop('Description') raw_role.pop('MaxSessionDuration') return raw_role['id'], raw_role From adb9475700feadf8fd159d5758dae8ef314a9c9e Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 18 Mar 2019 11:50:59 -0400 Subject: [PATCH 453/667] Remove dead code about elb_policies. --- ScoutSuite/providers/aws/provider.py | 36 ---------------------------- 1 file changed, 36 deletions(-) diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index e3da8c973..e634dacd1 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -61,7 +61,6 @@ def preprocessing(self, ip_ranges=None, ip_ranges_name_key=None): ip_ranges = [] if ip_ranges is None else ip_ranges self._map_all_subnets() self._set_emr_vpc_ids() - # self.parse_elb_policies() # Various data processing calls if 'ec2' in self.service_list: @@ -555,41 +554,6 @@ def set_emr_vpc_ids_callback(self, current_config, path, current_path, vpc_id, c if len(current_config['clusters']) == 0: callback_args['clear_list'].append(region) - def _parse_elb_policies(self): - if 'elb' in self.services: - self._go_to_and_do(self.services['elb'], - ['regions'], - [], - self.parse_elb_policies_callback, - {}) - - @staticmethod - def parse_elb_policies_callback(current_config, path, current_path, region_id, callback_args): - region_config = get_object_at(['services', 'elb', ] + current_path + [region_id]) - region_config['elb_policies'] = current_config['elb_policies'] - for policy_id in region_config['elb_policies']: - if region_config['elb_policies'][policy_id]['PolicyTypeName'] != 'SSLNegotiationPolicyType': - continue - # protocols, options, ciphers - policy = region_config['elb_policies'][policy_id] - protocols = {} - options = {} - ciphers = {} - for attribute in policy['PolicyAttributeDescriptions']: - if attribute['AttributeName'] in ['Protocol-SSLv3', 'Protocol-TLSv1', 'Protocol-TLSv1.1', - 'Protocol-TLSv1.2']: - protocols[attribute['AttributeName']] = attribute['AttributeValue'] - elif attribute['AttributeName'] in ['Server-Defined-Cipher-Order']: - options[attribute['AttributeName']] = attribute['AttributeValue'] - elif attribute['AttributeName'] == 'Reference-Security-Policy': - policy['reference_security_policy'] = attribute['AttributeValue'] - else: - ciphers[attribute['AttributeName']] = attribute['AttributeValue'] - policy['protocols'] = protocols - policy['options'] = options - policy['ciphers'] = ciphers - # TODO: pop ? - def sort_vpc_flow_logs_callback(self, current_config, path, current_path, flow_log_id, callback_args): attached_resource = current_config['ResourceId'] if attached_resource.startswith('vpc-'): From 7d38829f2758adbdaa2203bee523ec25d63f2cae Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 18 Mar 2019 12:11:40 -0400 Subject: [PATCH 454/667] Tried fixing the permissions --- .../providers/aws/resources/iam/service.py | 46 +++++++++++++------ 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/iam/service.py b/ScoutSuite/providers/aws/resources/iam/service.py index 502411e01..f3d7f58e3 100644 --- a/ScoutSuite/providers/aws/resources/iam/service.py +++ b/ScoutSuite/providers/aws/resources/iam/service.py @@ -41,19 +41,29 @@ def finalize(self): if 'attached_to' in policy and len(policy['attached_to']) > 0: for entity_type in policy['attached_to']: for entity in policy['attached_to'][entity_type]: - entity['id'] = self._get_id_for_resource(entity_type, entity['name']) + entity['id'] = self._get_id_for_resource( + entity_type, entity['name']) entities = self[entity_type] entities[entity['id']].setdefault('policies', []) entities[entity['id']].setdefault('policies_counts', 0) entities[entity['id']]['policies'].append(policy_id) entities[entity['id']]['policies_counts'] += 1 - self._parse_permissions(policy_id, policy['PolicyDocument'], 'policies', entity_type, entity['id']) + self._parse_permissions( + policy_id, policy['PolicyDocument'], 'policies', entity_type, entity['id']) else: - pass - self._parse_permissions(policy_id, policy['PolicyDocument'], 'policies', None, None) + self._parse_permissions( + policy_id, policy['PolicyDocument'], 'policies', None, None) def _get_inline_policies(self, resource_type): - return [resource['inline_policies'] for resource in self[resource_type].values() if 'inline_policies' in resource] + inline_policies = [] + for resource in self[resource_type].values(): + if 'inline_policies' not in resource: + continue + + for policy_id in resource['inline_policies']: + resource['inline_policies'][policy_id]['id'] = policy_id + inline_policies.append(resource['inline_policies'][policy_id]) + return inline_policies def _get_id_for_resource(self, iam_resource_type, resource_name): for resource_id in self[iam_resource_type]: @@ -92,10 +102,13 @@ def _parse_actions(self, effect, action_string, actions, resource_string, resour iam_resource_name, policy_name, policy_type, condition): for action in actions: self['permissions'][action_string].setdefault(action, {}) - self['permissions'][action_string][action].setdefault(iam_resource_type, {}) - self['permissions'][action_string][action][iam_resource_type].setdefault(effect, {}) - self['permissions'][action_string][action][iam_resource_type][effect].setdefault(iam_resource_name, {}) - self._parse_action(effect, action_string, action, resource_string, resources, iam_resource_type, \ + self['permissions'][action_string][action].setdefault( + iam_resource_type, {}) + self['permissions'][action_string][action][iam_resource_type].setdefault( + effect, {}) + self['permissions'][action_string][action][iam_resource_type][effect].setdefault( + iam_resource_name, {}) + self._parse_action(effect, action_string, action, resource_string, resources, iam_resource_type, iam_resource_name, policy_name, policy_type, condition) def _parse_action(self, effect, action_string, action, resource_string, resources, iam_resource_type, @@ -105,8 +118,13 @@ def _parse_action(self, effect, action_string, action, resource_string, resource iam_resource_name, policy_name, policy_type, condition) def _parse_resource(self, effect, action_string, action, resource_string, resource, iam_resource_type, iam_resource_name, policy_name, policy_type, condition): - self['permissions'][action_string][action][iam_resource_type][effect][iam_resource_name].setdefault(resource_string, {}) - self['permissions'][action_string][action][iam_resource_type][effect][iam_resource_name][resource_string].setdefault(resource, {}) - self['permissions'][action_string][action][iam_resource_type][effect][iam_resource_name][resource_string][resource].setdefault(policy_type, {}) - self['permissions'][action_string][action][iam_resource_type][effect][iam_resource_name][resource_string][resource][policy_type].setdefault(policy_name, {}) - self['permissions'][action_string][action][iam_resource_type][effect][iam_resource_name][resource_string][resource][policy_type][policy_name].setdefault('condition', condition) + self['permissions'][action_string][action][iam_resource_type][effect][iam_resource_name].setdefault( + resource_string, {}) + self['permissions'][action_string][action][iam_resource_type][effect][iam_resource_name][resource_string].setdefault(resource, { + }) + self['permissions'][action_string][action][iam_resource_type][effect][iam_resource_name][resource_string][resource].setdefault( + policy_type, {}) + self['permissions'][action_string][action][iam_resource_type][effect][ + iam_resource_name][resource_string][resource][policy_type].setdefault(policy_name, {}) + self['permissions'][action_string][action][iam_resource_type][effect][iam_resource_name][ + resource_string][resource][policy_type][policy_name].setdefault('condition', condition) From e6e0d1bd241fe2554d3bc3ddc7d46af04bda1962 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 18 Mar 2019 12:38:33 -0400 Subject: [PATCH 455/667] Move attributes fetching to ELB facade and make it concurrent. --- ScoutSuite/providers/aws/facade/elb.py | 19 ++++++++++++++----- .../aws/resources/elb/load_balancers.py | 11 ++++------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/elb.py b/ScoutSuite/providers/aws/facade/elb.py index 9523b2ae5..3ad1072b7 100644 --- a/ScoutSuite/providers/aws/facade/elb.py +++ b/ScoutSuite/providers/aws/facade/elb.py @@ -3,7 +3,7 @@ from ScoutSuite.providers.aws.utils import ec2_classic from ScoutSuite.providers.utils import run_concurrently -from asyncio import Lock +import asyncio class ELBFacade(AWSBaseFacade): @@ -15,7 +15,7 @@ async def get_load_balancers(self, region: str, vpc: str): return [load_balancer for load_balancer in self.load_balancers_cache[region] if load_balancer['VpcId'] == vpc] async def cache_load_balancers(self, region): - async with self.regional_load_balancers_cache_locks.setdefault(region, Lock()): + async with self.regional_load_balancers_cache_locks.setdefault(region, asyncio.Lock()): if region in self.load_balancers_cache: return @@ -27,9 +27,18 @@ async def cache_load_balancers(self, region): load_balancer['VpcId'] =\ load_balancer['VPCId'] if 'VPCId' in load_balancer and load_balancer['VPCId'] else ec2_classic - async def get_load_balancer_attributes(self, region:str, load_balancer: str): + if len(self.load_balancers_cache[region]) > 0: + # Fetch and set the attributes of all the load balancers concurrently: + tasks = { + asyncio.ensure_future( + self._get_and_set_load_balancer_attributes(region, load_balancer) + ) for load_balancer in self.load_balancers_cache[region] + } + await asyncio.wait(tasks) + + async def _get_and_set_load_balancer_attributes(self, region: str, load_balancer: {}): elb_client = AWSFacadeUtils.get_client('elb', region, self.session) - return await run_concurrently( + load_balancer['attributes'] = await run_concurrently( lambda: elb_client.describe_load_balancer_attributes( - LoadBalancerName=load_balancer)['LoadBalancerAttributes'] + LoadBalancerName=load_balancer['LoadBalancerName'])['LoadBalancerAttributes'] ) diff --git a/ScoutSuite/providers/aws/resources/elb/load_balancers.py b/ScoutSuite/providers/aws/resources/elb/load_balancers.py index 17d90897e..dad3b89e7 100644 --- a/ScoutSuite/providers/aws/resources/elb/load_balancers.py +++ b/ScoutSuite/providers/aws/resources/elb/load_balancers.py @@ -6,14 +6,14 @@ class LoadBalancers(AWSResources): async def fetch_all(self, **kwargs): raw_load_balancers = await self.facade.elb.get_load_balancers(self.scope['region'], self.scope['vpc']) - # TODO: parse is async, parallelize the following loop: for raw_load_balancer in raw_load_balancers: - id, load_balancer = await self._parse_load_balancer(raw_load_balancer) + id, load_balancer = self._parse_load_balancer(raw_load_balancer) self[id] = load_balancer - async def _parse_load_balancer(self, raw_load_balancer): + def _parse_load_balancer(self, raw_load_balancer): load_balancer = {'name': raw_load_balancer.pop('LoadBalancerName')} - get_keys(raw_load_balancer, load_balancer, ['DNSName', 'CreatedTime', 'AvailabilityZones', 'Subnets', 'Scheme']) + get_keys(raw_load_balancer, load_balancer, + ['DNSName', 'CreatedTime', 'AvailabilityZones', 'Subnets', 'Scheme', 'attributes']) load_balancer['security_groups'] = [] for sg in raw_load_balancer['SecurityGroups']: @@ -28,7 +28,4 @@ async def _parse_load_balancer(self, raw_load_balancer): for i in raw_load_balancer['Instances']: load_balancer['instances'].append(i['InstanceId']) - load_balancer['attributes'] =\ - await self.facade.elb.get_load_balancer_attributes(self.scope['region'], load_balancer['name']) - return get_non_provider_id(load_balancer['name']), load_balancer From a145a0f2064dab99171545f2e4f02a0274373cac Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 18 Mar 2019 14:36:41 -0400 Subject: [PATCH 456/667] Implemented password policy node --- .../aws/resources/iam/passwordpolicy.py | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 ScoutSuite/providers/aws/resources/iam/passwordpolicy.py diff --git a/ScoutSuite/providers/aws/resources/iam/passwordpolicy.py b/ScoutSuite/providers/aws/resources/iam/passwordpolicy.py new file mode 100644 index 000000000..cfbf85d3a --- /dev/null +++ b/ScoutSuite/providers/aws/resources/iam/passwordpolicy.py @@ -0,0 +1,33 @@ +from ScoutSuite.providers.aws.resources.resources import AWSResources +from botocore.exceptions import ClientError + + +class PasswordPolicy(AWSResources): + async def fetch_all(self, **kwargs): + try: + password_policy = self._parse_password_policy(await self.facade.iam.get_password_policy()) + except ClientError as e: + if e.response['Error']['Code'] == 'NoSuchEntity': + password_policy = { + 'MinimumPasswordLength': '1', + 'RequireUppercaseCharacters': False, + 'RequireLowercaseCharacters': False, 'RequireNumbers': False, + 'RequireSymbols': False, 'PasswordReusePrevention': False, + 'ExpirePasswords': False + } + else: + raise e + + self.update(password_policy) + + def _parse_password_policy(self, raw_password_policy): + if 'PasswordReusePrevention' not in raw_password_policy: + raw_password_policy['PasswordReusePrevention'] = False + else: + raw_password_policy['PreviousPasswordPrevented'] = raw_password_policy['PasswordReusePrevention'] + raw_password_policy['PasswordReusePrevention'] = True + # There is a bug in the API: ExpirePasswords always returns false + if 'MaxPasswordAge' in raw_password_policy: + raw_password_policy['ExpirePasswords'] = True + + return raw_password_policy From c5e6ef72dc1586b44ef738c6f948bfb2d04a8a69 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 18 Mar 2019 14:36:56 -0400 Subject: [PATCH 457/667] Integrated password policy --- ScoutSuite/providers/aws/resources/iam/service.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/resources/iam/service.py b/ScoutSuite/providers/aws/resources/iam/service.py index f3d7f58e3..6f789d489 100644 --- a/ScoutSuite/providers/aws/resources/iam/service.py +++ b/ScoutSuite/providers/aws/resources/iam/service.py @@ -15,7 +15,8 @@ class IAM(AWSCompositeResources): (Groups, 'groups'), (Policies, 'policies'), (Users, 'users'), - (Roles, 'roles') + (Roles, 'roles'), + (PasswordPolicy, 'password_policy') ] def __init__(self): From 8cb3d52fa4115934453ac63c66691a3cb01797ee Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 18 Mar 2019 14:37:18 -0400 Subject: [PATCH 458/667] Added get password policy to facade --- ScoutSuite/providers/aws/facade/iam.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ScoutSuite/providers/aws/facade/iam.py b/ScoutSuite/providers/aws/facade/iam.py index d3204e7a5..04dee60e9 100644 --- a/ScoutSuite/providers/aws/facade/iam.py +++ b/ScoutSuite/providers/aws/facade/iam.py @@ -123,6 +123,11 @@ async def get_roles(self): return roles + async def get_password_policy(self): + client = AWSFacadeUtils.get_client('iam', self.session) + response = await run_concurrently(lambda: client.get_account_password_policy()) + return response['PasswordPolicy'] + async def _get_user_acces_keys(self, user_name): client = AWSFacadeUtils.get_client('iam', self.session) response = await run_concurrently(lambda: client.list_access_keys(UserName=user_name)) From dd0e1685a325324e8143c5feec77d52670b3f092 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 18 Mar 2019 14:37:23 -0400 Subject: [PATCH 459/667] Lint --- ScoutSuite/providers/aws/facade/iam.py | 42 ++++++++++++++++---------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/iam.py b/ScoutSuite/providers/aws/facade/iam.py index 04dee60e9..1f09fd42e 100644 --- a/ScoutSuite/providers/aws/facade/iam.py +++ b/ScoutSuite/providers/aws/facade/iam.py @@ -36,7 +36,8 @@ async def get_groups(self): groups = await AWSFacadeUtils.get_all_pages('iam', None, self.session, 'list_groups', 'Groups') for group in groups: group['Users'] = await self._fetch_group_users(group['GroupName']) - policies = self._get_inline_policies('group', group['GroupId'], group['GroupName']) + policies = self._get_inline_policies( + 'group', group['GroupId'], group['GroupName']) if len(policies): group['inline_policies'] = policies group['inline_policies_count'] = len(policies) @@ -48,7 +49,8 @@ async def get_policies(self): # TODO: Parallelize this for policy in policies: - policy_version = client.get_policy_version(PolicyArn=policy['Arn'], VersionId=policy['DefaultVersionId']) + policy_version = client.get_policy_version( + PolicyArn=policy['Arn'], VersionId=policy['DefaultVersionId']) policy['PolicyDocument'] = policy_version['PolicyVersion']['Document'] policy['attached_to'] = {} @@ -60,11 +62,13 @@ async def get_policies(self): policy['attached_to'][resource_type] = [] for entity in attached_entities[entity_type]: - name_field = entity_type.replace('Policy', '')[:-1] + 'Name' + name_field = entity_type.replace('Policy', '')[ + :-1] + 'Name' resource_name = entity[name_field] id_field = entity_type.replace('Policy', '')[:-1] + 'Id' resource_id = entity[id_field] - policy['attached_to'][resource_type].append({'name': resource_name, 'id': resource_id}) + policy['attached_to'][resource_type].append( + {'name': resource_name, 'id': resource_id}) return policies @@ -103,23 +107,27 @@ async def get_roles(self): role['instances_count'] = 'N/A' # Get role policies - policies = self._get_inline_policies('role', role['RoleId'], role['RoleName']) + policies = self._get_inline_policies( + 'role', role['RoleId'], role['RoleName']) if len(policies): role['inline_policies'] = policies role['inline_policies_count'] = len(policies) # Get instance profiles - profiles = await AWSFacadeUtils.get_all_pages('iam', None, self.session, 'list_instance_profiles_for_role', 'InstanceProfiles', RoleName = role['RoleName']) + profiles = await AWSFacadeUtils.get_all_pages('iam', None, self.session, 'list_instance_profiles_for_role', 'InstanceProfiles', RoleName=role['RoleName']) role.setdefault('instance_profiles', {}) for profile in profiles: profile_id = profile['InstanceProfileId'] role['instance_profiles'].setdefault(profile_id, {}) - role['instance_profiles'][profile_id].setdefault('arn', profile['Arn']) - role['instance_profiles'][profile_id].setdefault('name', profile['InstanceProfileName']) + role['instance_profiles'][profile_id].setdefault( + 'arn', profile['Arn']) + role['instance_profiles'][profile_id].setdefault( + 'name', profile['InstanceProfileName']) # Get trust relationship role['assume_role_policy'] = {} - role['assume_role_policy']['PolicyDocument'] = role.pop('AssumeRolePolicyDocument') + role['assume_role_policy']['PolicyDocument'] = role.pop( + 'AssumeRolePolicyDocument') return roles @@ -138,7 +146,6 @@ async def _get_user_mfa_devices(self, user_name): response = await run_concurrently(lambda: client.list_mfa_devices(UserName=user_name)) return response['MFADevices'] - async def _fetch_group_users(self, group_name): client = AWSFacadeUtils.get_client('iam', self.session) fetched_users = client.get_group(GroupName=group_name)['Users'] @@ -151,9 +158,11 @@ async def _fetch_group_users(self, group_name): # TODO: Make this async def _get_inline_policies(self, iam_resource_type, resource_id, resource_name): client = AWSFacadeUtils.get_client('iam', self.session) - get_policy_method = getattr(client, 'get_' + iam_resource_type + '_policy') + get_policy_method = getattr( + client, 'get_' + iam_resource_type + '_policy') fetched_policies = {} - list_policy_method = getattr(client, 'list_' + iam_resource_type + '_policies') + list_policy_method = getattr( + client, 'list_' + iam_resource_type + '_policies') args = {iam_resource_type.title() + 'Name': resource_name} try: policy_names = list_policy_method(**args)['PolicyNames'] @@ -169,7 +178,8 @@ def _get_inline_policies(self, iam_resource_type, resource_id, resource_name): policy_document = get_policy_method(**args)['PolicyDocument'] policy_id = get_non_provider_id(policy_name) fetched_policies[policy_id] = {} - fetched_policies[policy_id]['PolicyDocument'] = self._normalize_statements(policy_document) + fetched_policies[policy_id]['PolicyDocument'] = self._normalize_statements( + policy_document) fetched_policies[policy_id]['name'] = policy_name fetched_policies[policy_id] = fetched_policies[policy_id] except Exception as e: @@ -178,8 +188,8 @@ def _get_inline_policies(self, iam_resource_type, resource_id, resource_name): else: print_exception(e) return fetched_policies - - def _normalize_statements(self, policy_document) : + + def _normalize_statements(self, policy_document): for statement in policy_document['Statement']: # Action or NotAction action_string = 'Action' if 'Action' in statement else 'NotAction' @@ -190,4 +200,4 @@ def _normalize_statements(self, policy_document) : if type(statement[resource_string]) != list: statement[resource_string] = [statement[resource_string]] - return policy_document \ No newline at end of file + return policy_document From 29a836fe0b1eb9af959d2ac39f4e477dfb72e8db Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 18 Mar 2019 14:37:33 -0400 Subject: [PATCH 460/667] Fixed missing policies --- .../providers/aws/resources/iam/service.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/iam/service.py b/ScoutSuite/providers/aws/resources/iam/service.py index 6f789d489..9e812cc57 100644 --- a/ScoutSuite/providers/aws/resources/iam/service.py +++ b/ScoutSuite/providers/aws/resources/iam/service.py @@ -6,6 +6,7 @@ from ScoutSuite.providers.aws.resources.iam.policies import Policies from ScoutSuite.providers.aws.resources.iam.users import Users from ScoutSuite.providers.aws.resources.iam.roles import Roles +from ScoutSuite.providers.aws.resources.iam.passwordpolicy import PasswordPolicy from ScoutSuite.providers.aws.facade.facade import AWSFacade @@ -33,9 +34,9 @@ def finalize(self): # Update permissions for managed policies self['permissions'] = {} policies = [policy for policy in self['policies'].values()] - policies.extend(self._get_inline_policies('groups')) - policies.extend(self._get_inline_policies('users')) - policies.extend(self._get_inline_policies('roles')) + self._get_inline_policies('groups') + self._get_inline_policies('users') + self._get_inline_policies('roles') for policy in policies: policy_id = policy['id'] @@ -56,15 +57,15 @@ def finalize(self): policy_id, policy['PolicyDocument'], 'policies', None, None) def _get_inline_policies(self, resource_type): - inline_policies = [] - for resource in self[resource_type].values(): + for resource_id in self[resource_type]: + resource = self[resource_type][resource_id] if 'inline_policies' not in resource: continue for policy_id in resource['inline_policies']: - resource['inline_policies'][policy_id]['id'] = policy_id - inline_policies.append(resource['inline_policies'][policy_id]) - return inline_policies + policy = resource['inline_policies'][policy_id] + self._parse_permissions( + policy_id, policy['PolicyDocument'], 'inline_policies', resource_type, resource_id) def _get_id_for_resource(self, iam_resource_type, resource_name): for resource_id in self[iam_resource_type]: From c1df9840cc387909b571f39e28dc135d0e96700b Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 18 Mar 2019 14:48:47 -0400 Subject: [PATCH 461/667] Renamed method --- ScoutSuite/providers/aws/resources/iam/service.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/iam/service.py b/ScoutSuite/providers/aws/resources/iam/service.py index 9e812cc57..ba95f7c21 100644 --- a/ScoutSuite/providers/aws/resources/iam/service.py +++ b/ScoutSuite/providers/aws/resources/iam/service.py @@ -34,9 +34,9 @@ def finalize(self): # Update permissions for managed policies self['permissions'] = {} policies = [policy for policy in self['policies'].values()] - self._get_inline_policies('groups') - self._get_inline_policies('users') - self._get_inline_policies('roles') + self._parse_inline_policies_permissions('groups') + self._parse_inline_policies_permissions('users') + self._parse_inline_policies_permissions('roles') for policy in policies: policy_id = policy['id'] @@ -56,7 +56,7 @@ def finalize(self): self._parse_permissions( policy_id, policy['PolicyDocument'], 'policies', None, None) - def _get_inline_policies(self, resource_type): + def _parse_inline_policies_permissions(self, resource_type): for resource_id in self[resource_type]: resource = self[resource_type][resource_id] if 'inline_policies' not in resource: From 77322b3d864273c66d75a439f3c08b5740832b69 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 18 Mar 2019 15:26:01 -0400 Subject: [PATCH 462/667] Removed old implementation --- ScoutSuite/providers/aws/services/iam.py | 424 ----------------------- 1 file changed, 424 deletions(-) delete mode 100644 ScoutSuite/providers/aws/services/iam.py diff --git a/ScoutSuite/providers/aws/services/iam.py b/ScoutSuite/providers/aws/services/iam.py deleted file mode 100644 index 48ee8d7af..000000000 --- a/ScoutSuite/providers/aws/services/iam.py +++ /dev/null @@ -1,424 +0,0 @@ -# -*- coding: utf-8 -*- - -from botocore.exceptions import ClientError - -from ScoutSuite.core.console import print_error, print_exception -from ScoutSuite.providers.aws.aws import connect_service, handle_truncated_response -from ScoutSuite.providers.aws.configs.base import AWSBaseConfig -from ScoutSuite.providers.aws.utils import get_keys, is_throttled -from ScoutSuite.utils import * - - -# noinspection PyBroadException -class IAMConfig(AWSBaseConfig): - """ - Object that holds the IAM configuration - - :ivar credential_reports: Credential report as downloaded from the AWS API - :ivar groups: Dictionary of IAM groups in the AWS account - :ivar groups_count: len(groups) - :ivar password_policy: Account password policy - :ivar permissions: Summary of permissions granted via all IAM policies - :ivar policies: Dictionary of IAM managed policies in use within the AWS Account - :ivar policies_count: len(policies) - :ivar roles: Dictionary of IAM roles in the AWS account - :ivar roles_count: len(roles) - :ivar users: Dictionary of IAM users in the AWS account - :ivar users_count: len(users) - """ - - targets = ( - - ('groups', 'Groups', 'list_groups', {}, False), - ('policies', 'Policies', 'list_policies', [{'Scope': 'Local'}, {'OnlyAttached': True}], False), - ('roles', 'Roles', 'list_roles', {}, False), - ('users', 'Users', 'list_users', {}, False), - ('credential_reports', '', '', {}, False), - ('password_policy', '', '', {}, False) - # TODO: Federations - ) - - def __init__(self, target_config): - self.credential_reports = {} - self.groups = {} - self.password_policy = {} - self.permissions = {} - self.policies = {} - self.roles = {} - self.users = {} - super(IAMConfig, self).__init__(target_config) - - ######################################## - # Overload to fetch credentials report before and after - ######################################## - - def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): - regions = [] if regions is None else regions - self.fetch_credential_reports(credentials, True) - super(IAMConfig, self).fetch_all(credentials, regions, partition_name, targets) - self.fetch_password_policy(credentials) - self.fetch_credential_reports(credentials) - self.fetchstatuslogger.show(True) - - ######################################## - # Credential report - ######################################## - - def fetch_credential_reports(self, credentials, ignore_exception=False): - """ - Fetch the credential report - - :param: api_client - :type: FOO - :param: ignore_exception : initiate credential report creation as not always ready - :type: Boolean - """ - credential_reports = {} - try: - api_client = connect_service('iam', credentials, silent=True) - response = api_client.generate_credential_report() - if response['State'] != 'COMPLETE': - if not ignore_exception: - print_error('Failed to generate a credential report.') - return - - report = api_client.get_credential_report()['Content'] - lines = report.splitlines() - keys = lines[0].decode('utf-8').split(',') - self.fetchstatuslogger.counts['credential_reports']['discovered'] = len(lines) - 1 - - for line in lines[1:]: - credential_report = {} - values = line.decode('utf-8').split(',') - user_id = values[0] - for key, value in zip(keys, values): - credential_report[key] = value - - credential_report['password_last_used'] = self._sanitize_date(credential_report['password_last_used']) - credential_report['access_key_1_last_used_date'] = self._sanitize_date( - credential_report['access_key_1_last_used_date']) - credential_report['access_key_2_last_used_date'] = self._sanitize_date( - credential_report['access_key_2_last_used_date']) - credential_report['last_used'] = self._compute_last_used(credential_report) - credential_report['name'] = user_id - credential_report['id'] = user_id - manage_dictionary(credential_reports, user_id, credential_report) - self.fetchstatuslogger.counts['credential_reports']['fetched'] = len(credential_reports) - - self.credential_reports = credential_reports - - except Exception as e: - if ignore_exception: - return - print_error('Failed to download a credential report.') - print_exception(e) - - @staticmethod - def _compute_last_used(credential_report): - dates = [credential_report['password_last_used'], - credential_report['access_key_1_last_used_date'], - credential_report['access_key_2_last_used_date']] - - dates = [date for date in dates if date is not None] - return max(dates) if len(dates) > 0 else None - - @staticmethod - def _sanitize_date(date): - """ - Returns the date if it is not equal to 'N/A' or 'no_information', else returns None - """ - return date if date != 'no_information' and date != 'N/A' else None - - ######################################## - # Groups - ######################################## - - def parse_groups(self, group, params): - """ - Parse a single IAM group and fetch additional information - """ - # When resuming upon throttling error, skip if already fetched - if group['GroupName'] in self.groups: - return - api_client = params['api_client'] - # Ensure consistent attribute names across resource types - group['id'] = group.pop('GroupId') - group['name'] = group.pop('GroupName') - group['arn'] = group.pop('Arn') - # Get group's members - group['users'] = self.__fetch_group_users(api_client, group['name']) - # Get inline policies - policies = self.__get_inline_policies(api_client, 'group', group['id'], group['name']) - if len(policies): - group['inline_policies'] = policies - group['inline_policies_count'] = len(policies) - self.groups[group['id']] = group - - ######################################## - # Managed policies - ######################################## - - def parse_policies(self, fetched_policy, params): - """ - Parse a single IAM policy and fetch additional information - """ - api_client = params['api_client'] - policy = {'name': fetched_policy.pop('PolicyName'), 'id': fetched_policy.pop('PolicyId'), - 'arn': fetched_policy.pop('Arn')} - # Download version and document - policy_version = api_client.get_policy_version(PolicyArn=policy['arn'], - VersionId=fetched_policy['DefaultVersionId']) - policy_version = policy_version['PolicyVersion'] - policy['PolicyDocument'] = policy_version['Document'] - # Get attached IAM entities - policy['attached_to'] = {} - attached_entities = handle_truncated_response(api_client.list_entities_for_policy, {'PolicyArn': policy['arn']}, - ['PolicyGroups', 'PolicyRoles', 'PolicyUsers']) - for entity_type in attached_entities: - resource_type = entity_type.replace('Policy', '').lower() - if len(attached_entities[entity_type]): - policy['attached_to'][resource_type] = [] - for entity in attached_entities[entity_type]: - name_field = entity_type.replace('Policy', '')[:-1] + 'Name' - resource_name = entity[name_field] - policy['attached_to'][resource_type].append({'name': resource_name}) - # Save policy - self.policies[policy['id']] = policy - - ######################################## - # Password policy - ######################################## - - def fetch_password_policy(self, credentials): - """ - Fetch the password policy that applies to all IAM users within the AWS account - """ - self.fetchstatuslogger.counts['password_policy']['discovered'] = 0 - self.fetchstatuslogger.counts['password_policy']['fetched'] = 0 - try: - api_client = connect_service('iam', credentials, silent=True) - self.password_policy = api_client.get_account_password_policy()['PasswordPolicy'] - if 'PasswordReusePrevention' not in self.password_policy: - self.password_policy['PasswordReusePrevention'] = False - else: - self.password_policy['PreviousPasswordPrevented'] = self.password_policy['PasswordReusePrevention'] - self.password_policy['PasswordReusePrevention'] = True - # There is a bug in the API: ExpirePasswords always returns false - if 'MaxPasswordAge' in self.password_policy: - self.password_policy['ExpirePasswords'] = True - self.fetchstatuslogger.counts['password_policy']['discovered'] = 1 - self.fetchstatuslogger.counts['password_policy']['fetched'] = 1 - - except ClientError as e: - if e.response['Error']['Code'] == 'NoSuchEntity': - self.password_policy = {'MinimumPasswordLength': '1', 'RequireUppercaseCharacters': False, - 'RequireLowercaseCharacters': False, 'RequireNumbers': False, - 'RequireSymbols': False, 'PasswordReusePrevention': False, - 'ExpirePasswords': False} - else: - raise e - except Exception as e: - print_error(str(e)) - - ######################################## - # Roles - ######################################## - - def parse_roles(self, fetched_role, params): - """ - Parse a single IAM role and fetch additional data - """ - role = {'instances_count': 'N/A'} - # When resuming upon throttling error, skip if already fetched - if fetched_role['RoleName'] in self.roles: - return - api_client = params['api_client'] - # Ensure consistent attribute names across resource types - role['id'] = fetched_role.pop('RoleId') - role['name'] = fetched_role.pop('RoleName') - role['arn'] = fetched_role.pop('Arn') - # Get other attributes - get_keys(fetched_role, role, ['CreateDate', 'Path']) - # Get role policies - policies = self.__get_inline_policies(api_client, 'role', role['id'], role['name']) - if len(policies): - role['inline_policies'] = policies - role['inline_policies_count'] = len(policies) - # Get instance profiles - profiles = handle_truncated_response(api_client.list_instance_profiles_for_role, {'RoleName': role['name']}, - ['InstanceProfiles']) - manage_dictionary(role, 'instance_profiles', {}) - for profile in profiles['InstanceProfiles']: - manage_dictionary(role['instance_profiles'], profile['InstanceProfileId'], {}) - role['instance_profiles'][profile['InstanceProfileId']]['arn'] = profile['Arn'] - role['instance_profiles'][profile['InstanceProfileId']]['name'] = profile['InstanceProfileName'] - # Get trust relationship - role['assume_role_policy'] = {} - role['assume_role_policy']['PolicyDocument'] = fetched_role.pop('AssumeRolePolicyDocument') - # Save role - self.roles[role['id']] = role - - ######################################## - # Users - ######################################## - - def parse_users(self, user, params): - """ - Parse a single IAM user and fetch additional data - """ - if user['UserName'] in self.users: - return - api_client = params['api_client'] - # Ensure consistent attribute names across resource types - user['id'] = user.pop('UserId') - user['name'] = user.pop('UserName') - user['arn'] = user.pop('Arn') - policies = self.__get_inline_policies(api_client, 'user', user['id'], user['name']) - if len(policies): - user['inline_policies'] = policies - user['inline_policies_count'] = len(policies) - user['groups'] = [] - groups = handle_truncated_response(api_client.list_groups_for_user, {'UserName': user['name']}, ['Groups'])[ - 'Groups'] - for group in groups: - user['groups'].append(group['GroupName']) - try: - user['LoginProfile'] = api_client.get_login_profile(UserName=user['name'])['LoginProfile'] - except Exception: - pass - user['AccessKeys'] = api_client.list_access_keys(UserName=user['name'])['AccessKeyMetadata'] - user['MFADevices'] = api_client.list_mfa_devices(UserName=user['name'])['MFADevices'] - # TODO: Users signing certss - self.users[user['id']] = user - - ######################################## - # Finalize IAM config - ######################################## - - def finalize(self): - # Update permissions for managed policies - for policy_id in self.policies: - if 'attached_to' in self.policies[policy_id] and len(self.policies[policy_id]['attached_to']) > 0: - for entity_type in self.policies[policy_id]['attached_to']: - for entity in self.policies[policy_id]['attached_to'][entity_type]: - entity['id'] = self.get_id_for_resource(entity_type, entity['name']) - entities = getattr(self, entity_type) - manage_dictionary(entities[entity['id']], 'policies', []) - manage_dictionary(entities[entity['id']], 'policies_counts', 0) - entities[entity['id']]['policies'].append(policy_id) - entities[entity['id']]['policies_counts'] += 1 - self.__parse_permissions(policy_id, self.policies[policy_id]['PolicyDocument'], 'policies', - entity_type, entity['id']) - else: - self.__parse_permissions(policy_id, self.policies[policy_id]['PolicyDocument'], 'policies', None, None) - super(IAMConfig, self).finalize() - - ######################################## - # Class helpers - ######################################## - - def get_id_for_resource(self, iam_resource_type, resource_name): - for resource_id in getattr(self, iam_resource_type): - if getattr(self, iam_resource_type)[resource_id]['name'] == resource_name: - return resource_id - - @staticmethod - def __fetch_group_users(api_client, group_name): - users = [] - fetched_users = api_client.get_group(GroupName=group_name)['Users'] - for user in fetched_users: - users.append(user['UserId']) - return users - - ######################################## - # Inline policies - ######################################## - - def __get_inline_policies(self, api_client, iam_resource_type, resource_id, resource_name): - fetched_policies = {} - get_policy_method = getattr(api_client, 'get_' + iam_resource_type + '_policy') - list_policy_method = getattr(api_client, 'list_' + iam_resource_type + '_policies') - args = {iam_resource_type.title() + 'Name': resource_name} - try: - policy_names = list_policy_method(**args)['PolicyNames'] - except Exception as e: - if is_throttled(e): - raise e - else: - print_exception(e) - return fetched_policies - try: - for policy_name in policy_names: - args['PolicyName'] = policy_name - policy_document = get_policy_method(**args)['PolicyDocument'] - policy_id = self.get_non_provider_id(policy_name) - manage_dictionary(fetched_policies, policy_id, {}) - fetched_policies[policy_id]['PolicyDocument'] = policy_document - fetched_policies[policy_id]['name'] = policy_name - self.__parse_permissions(policy_id, policy_document, 'inline_policies', iam_resource_type + 's', - resource_id) - except Exception as e: - if is_throttled(e): - raise e - else: - print_exception(e) - return fetched_policies - - def __parse_permissions(self, policy_name, policy_document, policy_type, iam_resource_type, resource_name): - # Enforce list of statements (Github issue #99) - if type(policy_document['Statement']) != list: - policy_document['Statement'] = [policy_document['Statement']] - for statement in policy_document['Statement']: - self.__parse_statement(policy_name, statement, policy_type, iam_resource_type, resource_name) - - def __parse_statement(self, policy_name, statement, policy_type, iam_resource_type, resource_name): - # Effect - effect = str(statement['Effect']) - # Action or NotAction - action_string = 'Action' if 'Action' in statement else 'NotAction' - if type(statement[action_string]) != list: - statement[action_string] = [statement[action_string]] - # Resource or NotResource - resource_string = 'Resource' if 'Resource' in statement else 'NotResource' - if type(statement[resource_string]) != list: - statement[resource_string] = [statement[resource_string]] - # Condition - condition = statement['Condition'] if 'Condition' in statement else None - manage_dictionary(self.permissions, action_string, {}) - if iam_resource_type is None: - return - self.__parse_actions(effect, action_string, statement[action_string], resource_string, - statement[resource_string], iam_resource_type, resource_name, policy_name, policy_type, - condition) - - def __parse_actions(self, effect, action_string, actions, resource_string, resources, iam_resource_type, - iam_resource_name, policy_name, policy_type, condition): - for action in actions: - manage_dictionary(self.permissions[action_string], action, {}) - manage_dictionary(self.permissions[action_string][action], iam_resource_type, {}) - manage_dictionary(self.permissions[action_string][action][iam_resource_type], effect, {}) - manage_dictionary(self.permissions[action_string][action][iam_resource_type][effect], iam_resource_name, {}) - self.__parse_action(effect, action_string, action, resource_string, resources, iam_resource_type, - iam_resource_name, policy_name, policy_type, condition) - - def __parse_action(self, effect, action_string, action, resource_string, resources, iam_resource_type, - iam_resource_name, policy_name, policy_type, condition): - for resource in resources: - self.__parse_resource(effect, action_string, action, resource_string, resource, iam_resource_type, - iam_resource_name, policy_name, policy_type, condition) - - def __parse_resource(self, effect, action_string, action, resource_string, resource, iam_resource_type, - iam_resource_name, policy_name, policy_type, condition): - manage_dictionary(self.permissions[action_string][action][iam_resource_type][effect][iam_resource_name], - resource_string, {}) - manage_dictionary( - self.permissions[action_string][action][iam_resource_type][effect][iam_resource_name][resource_string], - resource, {}) - manage_dictionary( - self.permissions[action_string][action][iam_resource_type][effect][iam_resource_name][resource_string][ - resource], policy_type, {}) - manage_dictionary( - self.permissions[action_string][action][iam_resource_type][effect][iam_resource_name][resource_string][ - resource][policy_type], policy_name, {}) - self.permissions[action_string][action][iam_resource_type][effect][iam_resource_name][resource_string][ - resource][policy_type][policy_name]['condition'] = condition From a9d8c4fbf899f4c71c157b3b063e1e9f67603a98 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 18 Mar 2019 15:26:19 -0400 Subject: [PATCH 463/667] Fixed sneaky indentation problem that caused some permissions to be missing --- ScoutSuite/providers/aws/resources/iam/service.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/iam/service.py b/ScoutSuite/providers/aws/resources/iam/service.py index ba95f7c21..8fdebb81e 100644 --- a/ScoutSuite/providers/aws/resources/iam/service.py +++ b/ScoutSuite/providers/aws/resources/iam/service.py @@ -50,8 +50,7 @@ def finalize(self): entities[entity['id']].setdefault('policies_counts', 0) entities[entity['id']]['policies'].append(policy_id) entities[entity['id']]['policies_counts'] += 1 - self._parse_permissions( - policy_id, policy['PolicyDocument'], 'policies', entity_type, entity['id']) + self._parse_permissions(policy_id, policy['PolicyDocument'], 'policies', entity_type, entity['id']) else: self._parse_permissions( policy_id, policy['PolicyDocument'], 'policies', None, None) From bb7b62ec4d1b1b79f5f2d3a093ba46c7450dfe3a Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 18 Mar 2019 15:26:28 -0400 Subject: [PATCH 464/667] Made finalize async --- ScoutSuite/providers/aws/resources/iam/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/resources/iam/service.py b/ScoutSuite/providers/aws/resources/iam/service.py index 8fdebb81e..817bb9c9f 100644 --- a/ScoutSuite/providers/aws/resources/iam/service.py +++ b/ScoutSuite/providers/aws/resources/iam/service.py @@ -30,7 +30,7 @@ async def fetch_all(self, credentials, regions=None, partition_name='aws'): self.facade._set_session(credentials) await self._fetch_children(self, {}) - def finalize(self): + async def finalize(self): # Update permissions for managed policies self['permissions'] = {} policies = [policy for policy in self['policies'].values()] From 7d4cf6ce906cb951de5a7c05825639e262837b3f Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 18 Mar 2019 15:27:52 -0400 Subject: [PATCH 465/667] Reset password policy count to zero --- ScoutSuite/providers/aws/resources/iam/service.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ScoutSuite/providers/aws/resources/iam/service.py b/ScoutSuite/providers/aws/resources/iam/service.py index 817bb9c9f..7b92edadb 100644 --- a/ScoutSuite/providers/aws/resources/iam/service.py +++ b/ScoutSuite/providers/aws/resources/iam/service.py @@ -30,6 +30,9 @@ async def fetch_all(self, credentials, regions=None, partition_name='aws'): self.facade._set_session(credentials) await self._fetch_children(self, {}) + # We do not want the report to count the password policies as resources, they aren't really resources. + self['password_policy_count'] = 0 + async def finalize(self): # Update permissions for managed policies self['permissions'] = {} From 89b036f0c65777a8af79fee2498d2aab58d0c408 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 18 Mar 2019 15:56:27 -0400 Subject: [PATCH 466/667] Define a facade for SNS service. --- ScoutSuite/providers/aws/facade/sns.py | 49 ++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 ScoutSuite/providers/aws/facade/sns.py diff --git a/ScoutSuite/providers/aws/facade/sns.py b/ScoutSuite/providers/aws/facade/sns.py new file mode 100644 index 000000000..96fb58869 --- /dev/null +++ b/ScoutSuite/providers/aws/facade/sns.py @@ -0,0 +1,49 @@ +from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils +from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade +from ScoutSuite.providers.utils import run_concurrently + +import asyncio + + +class SNSFacade(AWSBaseFacade): + regional_subscriptions_cache_locks = {} + subscriptions_cache = {} + + async def get_topics(self, region: str): + topics = await AWSFacadeUtils.get_all_pages('sns', region, self.session, 'list_topics', 'Topics') + + if len(topics) == 0: + return [] + + # Fetch and set the attributes of all topics concurrently: + tasks = { + asyncio.ensure_future( + self.get_and_set_topic_attributes(region, topic) + ) for topic in topics + } + await asyncio.wait(tasks) + + return topics + + async def get_and_set_topic_attributes(self, region: str, topic: {}): + sns_client = AWSFacadeUtils.get_client('sns', region, self.session) + topic['attributes'] = await run_concurrently( + lambda: sns_client.get_topic_attributes(TopicArn=topic['TopicArn'])['Attributes'] + ) + + async def get_subscriptions(self, region: str, topic_name: str): + await self.cache_subscriptions(region) + return [subscription for subscription in self.subscriptions_cache[region] + if subscription['topic_name'] == topic_name] + + async def cache_subscriptions(self, region: str): + async with self.regional_subscriptions_cache_locks.setdefault(region, asyncio.Lock()): + if region in self.subscriptions_cache: + return + + self.subscriptions_cache[region] =\ + await AWSFacadeUtils.get_all_pages('sns', region, self.session, 'list_subscriptions', 'Subscriptions') + + for subscription in self.subscriptions_cache[region]: + topic_arn = subscription.pop('TopicArn') + subscription['topic_name'] = topic_arn.split(':')[-1] From 937624581efbb0d3888c81938b20b0688f62bd3e Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 18 Mar 2019 15:56:48 -0400 Subject: [PATCH 467/667] Add SNS facade to AWS facade. --- ScoutSuite/providers/aws/facade/facade.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/facade/facade.py b/ScoutSuite/providers/aws/facade/facade.py index 92d09fa63..60f665612 100644 --- a/ScoutSuite/providers/aws/facade/facade.py +++ b/ScoutSuite/providers/aws/facade/facade.py @@ -11,9 +11,9 @@ from ScoutSuite.providers.aws.facade.efs import EFSFacade from ScoutSuite.providers.aws.facade.elasticache import ElastiCacheFacade from ScoutSuite.providers.aws.facade.emr import EMRFacade +from ScoutSuite.providers.aws.facade.sns import SNSFacade from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade from ScoutSuite.providers.utils import run_concurrently -from ScoutSuite.core.console import print_error, print_debug class AWSFacade(AWSBaseFacade): @@ -29,6 +29,7 @@ def __init__(self, credentials: dict = None): self.efs = EFSFacade(self.session) self.elasticache = ElastiCacheFacade(self.session) self.emr = EMRFacade(self.session) + self.sns = SNSFacade(self.session) async def build_region_list(self, service: str, chosen_regions=None, partition_name='aws'): service = 'ec2containerservice' if service == 'ecs' else service @@ -67,3 +68,4 @@ def _set_session(self, credentials: dict): self.efs = EFSFacade(self.session) self.elasticache = ElastiCacheFacade(self.session) self.emr = EMRFacade(self.session) + self.sns = SNSFacade(self.session) From 58ada96200f1a160bab3cb9dea6ef3b8a6497317 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 18 Mar 2019 15:57:06 -0400 Subject: [PATCH 468/667] Define Subscriptions resources. --- .../aws/resources/sns/subscriptions.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 ScoutSuite/providers/aws/resources/sns/subscriptions.py diff --git a/ScoutSuite/providers/aws/resources/sns/subscriptions.py b/ScoutSuite/providers/aws/resources/sns/subscriptions.py new file mode 100644 index 000000000..0736e3f04 --- /dev/null +++ b/ScoutSuite/providers/aws/resources/sns/subscriptions.py @@ -0,0 +1,19 @@ +from ScoutSuite.providers.aws.resources.resources import AWSResources + + +class Subscriptions(AWSResources): + async def fetch_all(self, **kwargs): + raw_subscriptions = await self.facade.sns.get_subscriptions(self.scope['region'], self.scope['topic_name']) + self['protocol'] = {} + self['subscriptions_count'] = 0 + for raw_subscription in raw_subscriptions: + protocol, subscription = self._parse_subscription(raw_subscription) + if protocol in self['protocol']: + self['protocol'][protocol].append(subscription) + else: + self['protocol'][protocol] = [subscription] + self['subscriptions_count'] += 1 + + def _parse_subscription(self, raw_subscription): + protocol = raw_subscription.pop('Protocol') + return protocol, raw_subscription From 095dfb539725e76803f2d3ce472ab1be4163c5e5 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 18 Mar 2019 15:57:25 -0400 Subject: [PATCH 469/667] Define Topics resources. --- .../providers/aws/resources/sns/topics.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 ScoutSuite/providers/aws/resources/sns/topics.py diff --git a/ScoutSuite/providers/aws/resources/sns/topics.py b/ScoutSuite/providers/aws/resources/sns/topics.py new file mode 100644 index 000000000..e15740ac0 --- /dev/null +++ b/ScoutSuite/providers/aws/resources/sns/topics.py @@ -0,0 +1,36 @@ +from ScoutSuite.providers.aws.resources.resources import AWSCompositeResources + +from .subscriptions import Subscriptions + +import json + + +class Topics(AWSCompositeResources): + _children = [ + (Subscriptions, 'subscriptions') + ] + + async def fetch_all(self, **kwargs): + raw_topics = await self.facade.sns.get_topics(self.scope['region']) + # TODO: parallelize this async loop: + for raw_topic in raw_topics: + topic_name, topic = self._parse_topic(raw_topic) + await self._fetch_children( + parent=topic, + scope={'region': self.scope['region'], 'topic_name': topic_name} + ) + # Fix subscriptions count: + topic['subscriptions_count'] = topic['subscriptions'].pop('subscriptions_count') + self[topic_name] = topic + + def _parse_topic(self, raw_topic): + raw_topic['arn'] = raw_topic.pop('TopicArn') + raw_topic['name'] = raw_topic['arn'].split(':')[-1] + + attributes = raw_topic.pop('attributes') + for k in ['Owner', 'DisplayName']: + raw_topic[k] = attributes[k] if k in attributes else None + for k in ['Policy', 'DeliveryPolicy', 'EffectiveDeliveryPolicy']: + raw_topic[k] = json.loads(attributes[k]) if k in attributes else None + + return raw_topic['name'], raw_topic From 4d3d99bec0e4de3e15b07ab2a1690e4e406cb90b Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 18 Mar 2019 15:57:39 -0400 Subject: [PATCH 470/667] Define SNS service. --- ScoutSuite/providers/aws/resources/sns/service.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 ScoutSuite/providers/aws/resources/sns/service.py diff --git a/ScoutSuite/providers/aws/resources/sns/service.py b/ScoutSuite/providers/aws/resources/sns/service.py new file mode 100644 index 000000000..66d1b38af --- /dev/null +++ b/ScoutSuite/providers/aws/resources/sns/service.py @@ -0,0 +1,12 @@ +from ScoutSuite.providers.aws.resources.regions import Regions + +from .topics import Topics + + +class SNS(Regions): + _children = [ + (Topics, 'topics') + ] + + def __init__(self): + super(SNS, self).__init__('sns') From d8c242a223ce856a7d36a54c4d44e3e63f30425c Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 18 Mar 2019 15:58:33 -0400 Subject: [PATCH 471/667] Migrate SNS service to the new architecture. --- ScoutSuite/providers/aws/configs/services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index 2c5311aef..9c07a3733 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -17,7 +17,7 @@ from ScoutSuite.providers.aws.services.route53 import Route53Config, Route53DomainsConfig from ScoutSuite.providers.aws.services.s3 import S3Config from ScoutSuite.providers.aws.services.ses import SESConfig -from ScoutSuite.providers.aws.services.sns import SNSConfig +from ScoutSuite.providers.aws.resources.sns.service import SNS from ScoutSuite.providers.aws.services.sqs import SQSConfig from ScoutSuite.providers.aws.services.vpc import VPCConfig from ScoutSuite.providers.base.configs.services import BaseServicesConfig @@ -72,7 +72,7 @@ def __init__(self, metadata=None, thread_config=4, **kwargs): self.route53domains = Route53DomainsConfig(thread_config) self.s3 = S3Config(thread_config) self.ses = SESConfig(metadata['messaging']['ses'], thread_config) - self.sns = SNSConfig(metadata['messaging']['sns'], thread_config) + self.sns = SNS() self.sqs = SQSConfig(metadata['messaging']['sqs'], thread_config) self.vpc = VPCConfig(metadata['network']['vpc'], thread_config) From 9ce4ac3d7d869178d4b07eeaadaca05f517894f3 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 18 Mar 2019 15:59:04 -0400 Subject: [PATCH 472/667] Remove old SNS service support. --- ScoutSuite/providers/aws/metadata.json | 8 --- ScoutSuite/providers/aws/services/sns.py | 75 ------------------------ 2 files changed, 83 deletions(-) delete mode 100644 ScoutSuite/providers/aws/services/sns.py diff --git a/ScoutSuite/providers/aws/metadata.json b/ScoutSuite/providers/aws/metadata.json index 0b4c2b7bd..b9797fe94 100644 --- a/ScoutSuite/providers/aws/metadata.json +++ b/ScoutSuite/providers/aws/metadata.json @@ -77,16 +77,8 @@ "sns": { "resources": { "topics": { - "api_call": "list_topics", - "response": "Topics", "cols": 2, "path": "services.sns.regions.id.topics" - }, - "subscriptions": { - "api_call": "list_subscriptions", - "response": "Subscriptions", - "hidden": true, - "path": "services.sns.regions.id.subscriptions" } } }, diff --git a/ScoutSuite/providers/aws/services/sns.py b/ScoutSuite/providers/aws/services/sns.py deleted file mode 100644 index a8413e939..000000000 --- a/ScoutSuite/providers/aws/services/sns.py +++ /dev/null @@ -1,75 +0,0 @@ -# -*- coding: utf-8 -*- -""" -SNS-related classes and functions -""" - -import json - -from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients -from ScoutSuite.utils import manage_dictionary - - -######################################## -# SNSRegionConfig -######################################## - -class SNSRegionConfig(RegionConfig): - """ - SNS configuration for a single AWS region - """ - topics = {} - - def parse_subscription(self, params, region, subscription): - """ - Parse a single subscription and reference it in its corresponding topic - - :param region: - :param params: Global parameters (defaults to {}) - :param subscription: SNS Subscription - """ - topic_arn = subscription.pop('TopicArn') - topic_name = topic_arn.split(':')[-1] - if topic_name in self.topics: - topic = self.topics[topic_name] - manage_dictionary(topic['subscriptions'], 'protocol', {}) - protocol = subscription.pop('Protocol') - manage_dictionary(topic['subscriptions']['protocol'], protocol, []) - topic['subscriptions']['protocol'][protocol].append(subscription) - topic['subscriptions_count'] += 1 - - def parse_topic(self, params, region, topic): - """ - Parse a single topic and fetch additional attributes - - :param region: Name of the AWS region - :param params: Global parameters (defaults to {}) - :param topic: SNS Topic - """ - topic['arn'] = topic.pop('TopicArn') - topic['name'] = topic['arn'].split(':')[-1] - (prefix, partition, service, region, account, name) = topic['arn'].split(':') - api_client = api_clients[region] - attributes = api_client.get_topic_attributes(TopicArn=topic['arn'])['Attributes'] - for k in ['Owner', 'DisplayName']: - topic[k] = attributes[k] if k in attributes else None - for k in ['Policy', 'DeliveryPolicy', 'EffectiveDeliveryPolicy']: - topic[k] = json.loads(attributes[k]) if k in attributes else None - topic['name'] = topic['arn'].split(':')[-1] - manage_dictionary(topic, 'subscriptions', {}) - manage_dictionary(topic, 'subscriptions_count', 0) - self.topics[topic['name']] = topic - - -######################################## -# SNSConfig -######################################## - -class SNSConfig(RegionalServiceConfig): - """ - SNS configuration for all AWS regions - """ - - region_config_class = SNSRegionConfig - - def __init__(self, service_metadata, thread_config=4): - super(SNSConfig, self).__init__(service_metadata, thread_config) From 6884d2468d360b139417d25e78b6ca24d2fe90f3 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 18 Mar 2019 16:29:47 -0400 Subject: [PATCH 473/667] Remove unused HTML partials. --- .../services.elb.regions.id.elb_policies.html | 54 ------------------- ....regions.id.vpcsid.elbs.linked_policy.html | 10 ---- 2 files changed, 64 deletions(-) delete mode 100644 ScoutSuite/output/data/html/partials/aws/services.elb.regions.id.elb_policies.html delete mode 100644 ScoutSuite/output/data/html/partials/aws/services.elb.regions.id.vpcsid.elbs.linked_policy.html diff --git a/ScoutSuite/output/data/html/partials/aws/services.elb.regions.id.elb_policies.html b/ScoutSuite/output/data/html/partials/aws/services.elb.regions.id.elb_policies.html deleted file mode 100644 index 274799a63..000000000 --- a/ScoutSuite/output/data/html/partials/aws/services.elb.regions.id.elb_policies.html +++ /dev/null @@ -1,54 +0,0 @@ - - - - - - - - diff --git a/ScoutSuite/output/data/html/partials/aws/services.elb.regions.id.vpcsid.elbs.linked_policy.html b/ScoutSuite/output/data/html/partials/aws/services.elb.regions.id.vpcsid.elbs.linked_policy.html deleted file mode 100644 index 4ea17e846..000000000 --- a/ScoutSuite/output/data/html/partials/aws/services.elb.regions.id.vpcsid.elbs.linked_policy.html +++ /dev/null @@ -1,10 +0,0 @@ - - - - From 60c8c493071a09d371907b6d46eef3437e4c7984 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 18 Mar 2019 22:21:32 -0400 Subject: [PATCH 474/667] Removed spaces --- ScoutSuite/providers/aws/facade/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/facade/utils.py b/ScoutSuite/providers/aws/facade/utils.py index fd3d0aceb..bb9ce31d1 100644 --- a/ScoutSuite/providers/aws/facade/utils.py +++ b/ScoutSuite/providers/aws/facade/utils.py @@ -35,7 +35,7 @@ def _get_all_pages_from_paginator(paginator, entities: list): return resources @staticmethod - def get_client(service: str, session: boto3.session.Session, region: str = None): + def get_client(service: str, session: boto3.session.Session, region: str=None): """ Instantiates an AWS API client From b1291111ad5af8dba13e5c696817b55ca7e2ebf7 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 18 Mar 2019 22:26:11 -0400 Subject: [PATCH 475/667] Fixed bad merge --- ScoutSuite/providers/aws/configs/services.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index 34bccb963..46edd483b 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -11,7 +11,6 @@ from ScoutSuite.providers.aws.services.elb import ELBConfig from ScoutSuite.providers.aws.resources.elbv2.service import ELBv2 from ScoutSuite.providers.aws.resources.iam.service import IAM -from ScoutSuite.providers.aws.services.elbv2 import ELBv2Config from ScoutSuite.providers.aws.resources.emr.service import EMR from ScoutSuite.providers.aws.services.rds import RDSConfig from ScoutSuite.providers.aws.resources.redshift.service import Redshift From 3c1b6cd3dbd05132837328ab20dd66aac6c1c952 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 18 Mar 2019 22:26:26 -0400 Subject: [PATCH 476/667] Uncommented preprocessing stuff and added enabled check --- ScoutSuite/providers/aws/provider.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index 6b8b84f3c..61daf6cdc 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -76,9 +76,11 @@ def preprocessing(self, ip_ranges=None, ip_ranges_name_key=None): if 'elbv2' in self.service_list: self._add_security_group_data_to_elbv2() + if 's3' in self.service_list and 'iam' in self.service_list: + self._match_iam_policies_and_buckets() + self._add_cidr_display_name(ip_ranges, ip_ranges_name_key) self._merge_route53_and_route53domains() - # self._match_iam_policies_and_buckets() # TODO: Uncomment this! super(AWSProvider, self).preprocessing() From 4f42487e845b635a510546d1607033e296a58b95 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 19 Mar 2019 10:20:34 -0400 Subject: [PATCH 477/667] Started implementing S3 facade --- ScoutSuite/providers/aws/facade/s3.py | 93 +++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 ScoutSuite/providers/aws/facade/s3.py diff --git a/ScoutSuite/providers/aws/facade/s3.py b/ScoutSuite/providers/aws/facade/s3.py new file mode 100644 index 000000000..40b5abccc --- /dev/null +++ b/ScoutSuite/providers/aws/facade/s3.py @@ -0,0 +1,93 @@ +from asyncio import Lock +from botocore.exceptions import ClientError + +from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils +from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade +from ScoutSuite.providers.utils import run_concurrently +from ScoutSuite.providers.utils import get_non_provider_id +from ScoutSuite.core.console import print_error, print_exception + + +class S3Facade(AWSBaseFacade): + + async def get_buckets(self): + client = AWSFacadeUtils.get_client('s3', None, self.session) + buckets = (await run_concurrently(lambda: client.list_buckets()))['Buckets'] + + for bucket in buckets: + bucket_name = bucket['Name'] + bucket['region'] = await self._get_s3_bucket_location(bucket_name) + await self._set_s3_bucket_logging(bucket) + await self._set_s3_bucket_versioning(bucket) + await self._set_s3_bucket_webhosting(bucket) + await self._set_s3_bucket_default_encryption(bucket) + + return buckets + + async def _get_s3_bucket_location(self, bucket_name): + client = AWSFacadeUtils.get_client('s3', None, self.session) + location = client.get_bucket_location(Bucket=bucket_name) + region = location['LocationConstraint'] if location['LocationConstraint'] else 'us-east-1' + + # Fixes issue #59: location constraint can be either EU or eu-west-1 for Ireland... + if region == 'EU': + region = 'eu-west-1' + + return region + + async def _set_s3_bucket_logging(self, bucket): + client = AWSFacadeUtils.get_client('s3', bucket['region'], self.session) + try: + logging = await run_concurrently(lambda: client.get_bucket_logging(Bucket=bucket['Name'])) + except Exception as e: + print_error('Failed to get logging configuration for %s: %s' % (bucket['Name'], e)) + bucket['logging'] = 'Unknown' + + if 'LoggingEnabled' in logging: + bucket['logging'] = logging['LoggingEnabled']['TargetBucket'] + '/' + logging['LoggingEnabled']['TargetPrefix'] + else: + bucket['logging'] = 'Disabled' + + # noinspection PyBroadException + async def _set_s3_bucket_versioning(self, bucket): + client = AWSFacadeUtils.get_client('s3', bucket['region'], self.session) + try: + versioning = await run_concurrently(lambda: client.get_bucket_versioning(Bucket=bucket['Name'])) + bucket['versioning_status_enabled'] = self._status_to_bool(versioning.get('Status')) + bucket['version_mfa_delete_enabled'] = self._status_to_bool(versioning.get('MFADelete')) + except Exception: + bucket['versioning_status_enabled'] = None + bucket['version_mfa_delete_enabled'] = None + + + # noinspection PyBroadException + async def _set_s3_bucket_webhosting(self, bucket): + client = AWSFacadeUtils.get_client('s3', bucket['region'], self.session) + try: + result = client.get_bucket_website(Bucket=bucket['Name']) + bucket['web_hosting_enabled'] = 'IndexDocument' in result + except Exception: + # TODO: distinguish permission denied from 'NoSuchWebsiteConfiguration' errors + bucket['web_hosting_enabled'] = False + + async def _set_s3_bucket_default_encryption(self, bucket): + bucket_name = bucket['Name'] + client = AWSFacadeUtils.get_client('s3', bucket['region'], self.session) + try: + await run_concurrently(lambda: client.get_bucket_encryption(Bucket=bucket['Name'])) + bucket['default_encryption_enabled'] = True + except ClientError as e: + if 'ServerSideEncryptionConfigurationNotFoundError' in e.response['Error']['Code']: + bucket['default_encryption_enabled'] = False + else: + print_error('Failed to get encryption configuration for %s: %s' % (bucket_name, e)) + bucket['default_encryption_enabled'] = None + except Exception as e: + print_error('Failed to get encryption configuration for %s: %s' % (bucket_name, e)) + bucket['default_encryption'] = 'Unknown' + + @staticmethod + def _status_to_bool(value): + """ Converts a string to True if it is equal to 'Enabled' or to False otherwise. """ + return value == 'Enabled' + From f6893371634bd030bed462cb5889307fc06c2a7d Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 19 Mar 2019 10:20:43 -0400 Subject: [PATCH 478/667] Integrated S3 facade to AWS facade --- ScoutSuite/providers/aws/facade/facade.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ScoutSuite/providers/aws/facade/facade.py b/ScoutSuite/providers/aws/facade/facade.py index 9d0649c20..91466b26c 100644 --- a/ScoutSuite/providers/aws/facade/facade.py +++ b/ScoutSuite/providers/aws/facade/facade.py @@ -13,6 +13,7 @@ from ScoutSuite.providers.aws.facade.emr import EMRFacade from ScoutSuite.providers.aws.facade.elbv2 import ELBv2Facade from ScoutSuite.providers.aws.facade.redshift import RedshiftFacade +from ScoutSuite.providers.aws.facade.s3 import S3Facade from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade from ScoutSuite.providers.utils import run_concurrently @@ -32,6 +33,7 @@ def __init__(self, credentials: dict = None): self.emr = EMRFacade(self.session) self.elbv2 = ELBv2Facade(self.session) self.redshift = RedshiftFacade(self.session) + self.s3 = S3Facade(self.session) async def build_region_list(self, service: str, chosen_regions=None, partition_name='aws'): service = 'ec2containerservice' if service == 'ecs' else service @@ -72,3 +74,4 @@ def _set_session(self, credentials: dict): self.emr = EMRFacade(self.session) self.elbv2 = ELBv2Facade(self.session) self.redshift = RedshiftFacade(self.session) + self.s3 = S3Facade(self.session) From d9ef880bb47b09d8e2c0e7e3f467155398c3d8c1 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 19 Mar 2019 10:20:59 -0400 Subject: [PATCH 479/667] Removed dead code --- ScoutSuite/providers/aws/services/s3.py | 89 ------------------------- 1 file changed, 89 deletions(-) diff --git a/ScoutSuite/providers/aws/services/s3.py b/ScoutSuite/providers/aws/services/s3.py index 3f0f4ec88..926dafae2 100644 --- a/ScoutSuite/providers/aws/services/s3.py +++ b/ScoutSuite/providers/aws/services/s3.py @@ -73,95 +73,6 @@ def parse_buckets(self, bucket, params): bucket['id'] = self.get_non_provider_id(bucket['name']) self.buckets[bucket['id']] = bucket - -def match_iam_policies_and_buckets(s3_info, iam_info): - if 'Action' in iam_info['permissions']: - for action in (x for x in iam_info['permissions']['Action'] if - ((x.startswith('s3:') and x != 's3:ListAllMyBuckets') or (x == '*'))): - for iam_entity in iam_info['permissions']['Action'][action]: - if 'Allow' in iam_info['permissions']['Action'][action][iam_entity]: - for allowed_iam_entity in iam_info['permissions']['Action'][action][iam_entity]['Allow']: - # For resource statements, we can easily rely on the existing permissions structure - if 'Resource' in \ - iam_info['permissions']['Action'][action][iam_entity]['Allow'][allowed_iam_entity]: - for full_path in (x for x in iam_info['permissions']['Action'][action][iam_entity]['Allow'][ - allowed_iam_entity]['Resource'] if x.startswith('arn:aws:s3:') or x == '*'): - parts = full_path.split('/') - bucket_name = parts[0].split(':')[-1] - update_iam_permissions(s3_info, bucket_name, iam_entity, allowed_iam_entity, - iam_info['permissions']['Action'][action][iam_entity]['Allow'][ - allowed_iam_entity]['Resource'][full_path]) - # For notresource statements, we must fetch the policy document to determine which buckets are - # not protected - if 'NotResource' in iam_info['permissions']['Action'][action][iam_entity]['Allow'][ - allowed_iam_entity]: - for full_path in (x for x in iam_info['permissions']['Action'][action][iam_entity]['Allow'][ - allowed_iam_entity]['NotResource'] if x.startswith('arn:aws:s3:') or x == '*'): - for policy_type in ['InlinePolicies', 'ManagedPolicies']: - if policy_type in iam_info['permissions']['Action'][action][iam_entity]['Allow'][ - allowed_iam_entity]['NotResource'][full_path]: - for policy in iam_info['permissions']['Action'][action][iam_entity]['Allow'][ - allowed_iam_entity]['NotResource'][full_path][policy_type]: - update_bucket_permissions(s3_info, iam_info, action, iam_entity, - allowed_iam_entity, full_path, policy_type, - policy) - - -def update_iam_permissions(s3_info, bucket_name, iam_entity, allowed_iam_entity, policy_info): - if bucket_name != '*' and bucket_name in s3_info['buckets']: - bucket = s3_info['buckets'][bucket_name] - manage_dictionary(bucket, iam_entity, {}) - manage_dictionary(bucket, iam_entity + '_count', 0) - if allowed_iam_entity not in bucket[iam_entity]: - bucket[iam_entity][allowed_iam_entity] = {} - bucket[iam_entity + '_count'] = bucket[iam_entity + '_count'] + 1 - - if 'inline_policies' in policy_info: - manage_dictionary(bucket[iam_entity][allowed_iam_entity], 'inline_policies', {}) - bucket[iam_entity][allowed_iam_entity]['inline_policies'].update(policy_info['inline_policies']) - if 'policies' in policy_info: - manage_dictionary(bucket[iam_entity][allowed_iam_entity], 'policies', {}) - bucket[iam_entity][allowed_iam_entity]['policies'].update(policy_info['policies']) - elif bucket_name == '*': - for bucket in s3_info['buckets']: - update_iam_permissions(s3_info, bucket, iam_entity, allowed_iam_entity, policy_info) - pass - else: - # Could be an error or cross-account access, ignore... - pass - - -def update_bucket_permissions(s3_info, iam_info, action, iam_entity, allowed_iam_entity, full_path, policy_type, - policy_name): - policy = {} - allowed_buckets = [] - # By default, all buckets are allowed - for bucket_name in s3_info['buckets']: - allowed_buckets.append(bucket_name) - if policy_type == 'InlinePolicies': - policy = iam_info[iam_entity.title()][allowed_iam_entity]['Policies'][policy_name]['PolicyDocument'] - elif policy_type == 'ManagedPolicies': - policy = iam_info['ManagedPolicies'][policy_name]['PolicyDocument'] - else: - print_error('Error, found unknown policy type.') - for statement in policy['Statement']: - for target_path in statement['NotResource']: - parts = target_path.split('/') - bucket_name = parts[0].split(':')[-1] - path = '/' + '/'.join(parts[1:]) if len(parts) > 1 else '/' - if (path == '/' or path == '/*') and (bucket_name in allowed_buckets): - # Remove bucket from list - allowed_buckets.remove(bucket_name) - elif bucket_name == '*': - allowed_buckets = [] - policy_info = {policy_type: {}} - policy_info[policy_type][policy_name] = \ - iam_info['permissions']['Action'][action][iam_entity]['Allow'][allowed_iam_entity]['NotResource'][full_path][ - policy_type][policy_name] - for bucket_name in allowed_buckets: - update_iam_permissions(s3_info, bucket_name, iam_entity, allowed_iam_entity, policy_info) - - def init_s3_permissions(): permissions = {'read': False, 'write': False, 'read_acp': False, 'write_acp': False} return permissions From fa1bba29503abdb4d8995148e861880864f97e59 Mon Sep 17 00:00:00 2001 From: misg Date: Tue, 19 Mar 2019 22:29:05 -0400 Subject: [PATCH 480/667] Define a facade for Route53 domains service. --- ScoutSuite/providers/aws/facade/route53.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 ScoutSuite/providers/aws/facade/route53.py diff --git a/ScoutSuite/providers/aws/facade/route53.py b/ScoutSuite/providers/aws/facade/route53.py new file mode 100644 index 000000000..740b713bb --- /dev/null +++ b/ScoutSuite/providers/aws/facade/route53.py @@ -0,0 +1,7 @@ +from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils +from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade + + +class Route53Facade(AWSBaseFacade): + async def get_domains(self, region): + return await AWSFacadeUtils.get_all_pages('route53domains', region, self.session, 'list_domains', 'Domains') From 5833bffc74e70da3006caf93b40b8775dfce2e7a Mon Sep 17 00:00:00 2001 From: misg Date: Tue, 19 Mar 2019 22:29:44 -0400 Subject: [PATCH 481/667] Add Route53 domains facade to AWS facade. --- ScoutSuite/providers/aws/facade/facade.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/facade/facade.py b/ScoutSuite/providers/aws/facade/facade.py index 92d09fa63..c3d4e4c56 100644 --- a/ScoutSuite/providers/aws/facade/facade.py +++ b/ScoutSuite/providers/aws/facade/facade.py @@ -11,9 +11,9 @@ from ScoutSuite.providers.aws.facade.efs import EFSFacade from ScoutSuite.providers.aws.facade.elasticache import ElastiCacheFacade from ScoutSuite.providers.aws.facade.emr import EMRFacade +from ScoutSuite.providers.aws.facade.route53 import Route53Facade from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade from ScoutSuite.providers.utils import run_concurrently -from ScoutSuite.core.console import print_error, print_debug class AWSFacade(AWSBaseFacade): @@ -29,6 +29,7 @@ def __init__(self, credentials: dict = None): self.efs = EFSFacade(self.session) self.elasticache = ElastiCacheFacade(self.session) self.emr = EMRFacade(self.session) + self.route53 = Route53Facade(self.session) async def build_region_list(self, service: str, chosen_regions=None, partition_name='aws'): service = 'ec2containerservice' if service == 'ecs' else service @@ -67,3 +68,4 @@ def _set_session(self, credentials: dict): self.efs = EFSFacade(self.session) self.elasticache = ElastiCacheFacade(self.session) self.emr = EMRFacade(self.session) + self.route53 = Route53Facade(self.session) From 0cfe4c49b7d6a01844bceee76832a313ef8bc229 Mon Sep 17 00:00:00 2001 From: misg Date: Tue, 19 Mar 2019 22:30:23 -0400 Subject: [PATCH 482/667] Define Route53Domains resources and Route53 service. --- .../aws/resources/route53/service.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 ScoutSuite/providers/aws/resources/route53/service.py diff --git a/ScoutSuite/providers/aws/resources/route53/service.py b/ScoutSuite/providers/aws/resources/route53/service.py new file mode 100644 index 000000000..a87a687da --- /dev/null +++ b/ScoutSuite/providers/aws/resources/route53/service.py @@ -0,0 +1,25 @@ +from ScoutSuite.providers.aws.resources.regions import Regions +from ScoutSuite.providers.aws.resources.resources import AWSResources +from ScoutSuite.providers.utils import get_non_provider_id + + +class Route53Domains(AWSResources): + async def fetch_all(self, **kwargs): + raw_domains = await self.facade.route53.get_domains(self.scope['region']) + for raw_domain in raw_domains: + id, domain = self._parse_domain(raw_domain) + self[id] = domain + + def _parse_domain(self, raw_domain): + domain_id = get_non_provider_id(raw_domain['DomainName']) + raw_domain['name'] = raw_domain.pop('DomainName') + return domain_id, raw_domain + + +class Route53(Regions): + _children = [ + (Route53Domains, 'domains') + ] + + def __init__(self): + super(Route53, self).__init__('route53domains') From 2316768dfc87b474e5ef88a7c41697e639eb456f Mon Sep 17 00:00:00 2001 From: misg Date: Tue, 19 Mar 2019 22:34:21 -0400 Subject: [PATCH 483/667] Migrate Route53 service to the new architecture. --- ScoutSuite/providers/aws/configs/services.py | 5 ++--- ScoutSuite/providers/aws/provider.py | 8 -------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index 2c5311aef..0c74d3823 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -14,7 +14,7 @@ from ScoutSuite.providers.aws.services.iam import IAMConfig from ScoutSuite.providers.aws.services.rds import RDSConfig from ScoutSuite.providers.aws.services.redshift import RedshiftConfig -from ScoutSuite.providers.aws.services.route53 import Route53Config, Route53DomainsConfig +from ScoutSuite.providers.aws.resources.route53.service import Route53 from ScoutSuite.providers.aws.services.s3 import S3Config from ScoutSuite.providers.aws.services.ses import SESConfig from ScoutSuite.providers.aws.services.sns import SNSConfig @@ -68,8 +68,7 @@ def __init__(self, metadata=None, thread_config=4, **kwargs): self.awslambda = Lambdas() self.redshift = RedshiftConfig(metadata['database']['redshift'], thread_config) self.rds = RDSConfig(metadata['database']['rds'], thread_config) - self.route53 = Route53Config(thread_config) - self.route53domains = Route53DomainsConfig(thread_config) + self.route53 = Route53() self.s3 = S3Config(thread_config) self.ses = SESConfig(metadata['messaging']['ses'], thread_config) self.sns = SNSConfig(metadata['messaging']['sns'], thread_config) diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index e3da8c973..90021cede 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -78,7 +78,6 @@ def preprocessing(self, ip_ranges=None, ip_ranges_name_key=None): self._add_security_group_data_to_elbv2() self._add_cidr_display_name(ip_ranges, ip_ranges_name_key) - self._merge_route53_and_route53domains() self._match_iam_policies_and_buckets() super(AWSProvider, self).preprocessing() @@ -500,13 +499,6 @@ def match_security_groups_and_resources_callback(self, current_config, path, cur print_error('Failed to parse %s in %s in %s' % (resource_type, vpc_id, region)) print_exception(e) - def _merge_route53_and_route53domains(self): - if 'route53domains' not in self.services: - return - # TODO: fix this - self.services['route53'].update(self.services['route53domains']) - self.services.pop('route53domains') - def _set_emr_vpc_ids(self): clear_list = [] self._go_to_and_do(self.services['emr'], From 36fc4fa2af9e86fa193ffc628449dc03b65874b7 Mon Sep 17 00:00:00 2001 From: misg Date: Tue, 19 Mar 2019 22:36:39 -0400 Subject: [PATCH 484/667] Remove old Route53 service support and unused hosted_zones resources. --- .../aws/services.route53.hosted_zones.html | 39 ----------- ScoutSuite/providers/aws/metadata.json | 6 +- ScoutSuite/providers/aws/services/route53.py | 66 ------------------- 3 files changed, 1 insertion(+), 110 deletions(-) delete mode 100644 ScoutSuite/output/data/html/partials/aws/services.route53.hosted_zones.html delete mode 100644 ScoutSuite/providers/aws/services/route53.py diff --git a/ScoutSuite/output/data/html/partials/aws/services.route53.hosted_zones.html b/ScoutSuite/output/data/html/partials/aws/services.route53.hosted_zones.html deleted file mode 100644 index 6bac579a9..000000000 --- a/ScoutSuite/output/data/html/partials/aws/services.route53.hosted_zones.html +++ /dev/null @@ -1,39 +0,0 @@ - - - - - - - - diff --git a/ScoutSuite/providers/aws/metadata.json b/ScoutSuite/providers/aws/metadata.json index 0b4c2b7bd..17fd62769 100644 --- a/ScoutSuite/providers/aws/metadata.json +++ b/ScoutSuite/providers/aws/metadata.json @@ -185,11 +185,7 @@ "resources": { "domains": { "cols": 2, - "path": "services.route53.domains" - }, - "hosted_zones": { - "cols": 2, - "path": "services.route53.hosted_zones" + "path": "services.route53.regions.id.domains" } } }, diff --git a/ScoutSuite/providers/aws/services/route53.py b/ScoutSuite/providers/aws/services/route53.py deleted file mode 100644 index e35945425..000000000 --- a/ScoutSuite/providers/aws/services/route53.py +++ /dev/null @@ -1,66 +0,0 @@ -# -*- coding: utf-8 -*- - -from ScoutSuite.providers.aws.utils import handle_truncated_response - -from ScoutSuite.providers.aws.configs.base import AWSBaseConfig - - -class Route53DomainsConfig(AWSBaseConfig): - """ - Object that holds the Route53Domains configuration - """ - - targets = ( - ('domains', 'Domains', 'list_domains', {}, False), - ) - - def __init__(self, target_config): - self.domains = {} - self.domains_count = 0 - super(Route53DomainsConfig, self).__init__(target_config) - - ######################################## - # Domains - ######################################## - def parse_domains(self, domain, params): - """ - Parse a single Route53Domains domain - """ - domain_id = self.get_non_provider_id(domain['DomainName']) - domain['name'] = domain.pop('DomainName') - # TODO: Get Dnssec info when available - # api_client = params['api_client'] - # details = api_client.get_domain_detail(DomainName = domain['name']) - # get_keys(details, domain, ['Dnssec']) - self.domains[domain_id] = domain - - -class Route53Config(AWSBaseConfig): - """ - Object that holds the Route53 configuration - """ - - targets = ( - ('hosted_zones', 'HostedZones', 'list_hosted_zones', {}, False), - ) - - def __init__(self, target_config): - self.hosted_zones = {} - self.hosted_zones_count = 0 - super(Route53Config, self).__init__(target_config) - - ######################################## - # Hosted_zoness - ######################################## - def parse_hosted_zones(self, hosted_zone, params): - """ - Parse a single Route53hosted_zoness hosted_zones - """ - # When resuming upon throttling error, skip if already fetched - hosted_zone_id = hosted_zone.pop('Id') - hosted_zone['name'] = hosted_zone.pop('Name') - api_client = params['api_client'] - record_sets = handle_truncated_response(api_client.list_resource_record_sets, {'HostedZoneId': hosted_zone_id}, - ['ResourceRecordSets']) - hosted_zone.update(record_sets) - self.hosted_zones[hosted_zone_id] = hosted_zone From fcc43363b67a03cbf53eee324a897e0eeb78abd0 Mon Sep 17 00:00:00 2001 From: misg Date: Tue, 19 Mar 2019 22:37:45 -0400 Subject: [PATCH 485/667] Fix resource paths. --- ...html => services.route53.regions.id.domains.html} | 12 ++++++------ .../rules/findings/route53-domain-no-autorenew.json | 2 +- .../findings/route53-domain-no-transferlock.json | 2 +- .../route53-domain-transferlock-not-authorized.json | 3 +-- 4 files changed, 9 insertions(+), 10 deletions(-) rename ScoutSuite/output/data/html/partials/aws/{services.route53.domains.html => services.route53.regions.id.domains.html} (61%) diff --git a/ScoutSuite/output/data/html/partials/aws/services.route53.domains.html b/ScoutSuite/output/data/html/partials/aws/services.route53.regions.id.domains.html similarity index 61% rename from ScoutSuite/output/data/html/partials/aws/services.route53.domains.html rename to ScoutSuite/output/data/html/partials/aws/services.route53.regions.id.domains.html index 5e516e302..b21771177 100644 --- a/ScoutSuite/output/data/html/partials/aws/services.route53.domains.html +++ b/ScoutSuite/output/data/html/partials/aws/services.route53.regions.id.domains.html @@ -1,15 +1,15 @@ - - - - - - - - - - diff --git a/ScoutSuite/output/data/html/partials/aws/services.vpc.regions.id.vpcs.id.peering_connections.html b/ScoutSuite/output/data/html/partials/aws/services.vpc.regions.id.vpcs.id.peering_connections.html deleted file mode 100644 index bd2982036..000000000 --- a/ScoutSuite/output/data/html/partials/aws/services.vpc.regions.id.vpcs.id.peering_connections.html +++ /dev/null @@ -1,12 +0,0 @@ - - - - diff --git a/ScoutSuite/providers/aws/metadata.json b/ScoutSuite/providers/aws/metadata.json index fdd424467..405455917 100644 --- a/ScoutSuite/providers/aws/metadata.json +++ b/ScoutSuite/providers/aws/metadata.json @@ -115,8 +115,6 @@ "vpc": { "resources": { "network_acls": { - "api_call": "describe_network_acls", - "response": "NetworkAcls", "cols": 2, "path": "services.vpc.regions.id.vpcs.id.network_acls", "callbacks": [ @@ -125,14 +123,10 @@ ] }, "vpcs": { - "api_call": "describe_vpcs", - "response": "Vpcs", "cols": 2, "path": "services.vpc.regions.id.vpcs" }, "flow_logs": { - "api_call": "describe_flow_logs", - "response": "FlowLogs", "hidden": true, "path": "services.vpc.regions.id.flow_logs", "callbacks": [ @@ -141,43 +135,8 @@ ] }, "subnets": { - "api_call": "describe_subnets", - "response": "Subnets", "cols": 2, "path": "services.vpc.regions.id.vpcs.id.subnets" - }, - "customer_gateways": { - "api_call": "describe_customer_gateways", - "response": "CustomerGateways", - "cols": 2, - "path": "services.vpc.regions.id.customer_gateways" - }, - "vpn_gateways": { - "api_call": "describe_vpn_gateways", - "response": "VpnGateways", - "cols": 2, - "path": "services.vpc.regions.id.vpn_gateways" - }, - "vpn_connections": { - "api_call": "describe_vpn_connections", - "response": "VpnConnections", - "cols": 2, - "path": "services.vpc.regions.id.vpn_connections" - }, - "route_tables": { - "api_call": "describe_route_tables", - "response": "RouteTables", - "hidden": true, - "path": "services.vpc.regions.id.vpcs.id.route_tables" - }, - "peering_connections": { - "api_call": "describe_vpc_peering_connections", - "response": "VpcPeeringConnections", - "hidden": true, - "path": "services.vpc.regions.id.peering_connections", - "callbacks": [ - [ "process_vpc_peering_connections_callback", {} ] - ] } } }, diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index 5e0ded86b..c40ff87f8 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -6,7 +6,7 @@ from ScoutSuite.core.console import print_debug, print_error, print_exception, print_info from ScoutSuite.providers.aws.configs.services import AWSServicesConfig -from ScoutSuite.providers.aws.services.vpc import put_cidr_name +from ScoutSuite.providers.aws.resources.vpc.service import put_cidr_name from ScoutSuite.providers.base.configs.browser import combine_paths, get_object_at, get_value_at from ScoutSuite.providers.base.provider import BaseProvider from ScoutSuite.utils import manage_dictionary @@ -420,33 +420,6 @@ def _get_role_info(self, attribute_name, attribute_value): break return iam_role_info - def process_vpc_peering_connections_callback(self, current_config, path, current_path, pc_id, callback_args): - - # Create a list of peering connection IDs in each VPC - info = 'AccepterVpcInfo' if current_config['AccepterVpcInfo'][ - 'OwnerId'] == self.aws_account_id else 'RequesterVpcInfo' - region = current_path[current_path.index('regions') + 1] - vpc_id = current_config[info]['VpcId'] - if vpc_id not in self.services['vpc']['regions'][region]['vpcs']: - region = current_config['AccepterVpcInfo']['Region'] - target = self.services['vpc']['regions'][region]['vpcs'][vpc_id] - else: - target = self.services['vpc']['regions'][region]['vpcs'][vpc_id] - manage_dictionary(target, 'peering_connections', []) - if pc_id not in target['peering_connections']: - target['peering_connections'].append(pc_id) - - # VPC information for the peer'd VPC - current_config['peer_info'] = copy.deepcopy( - current_config['AccepterVpcInfo' if info == 'RequesterVpcInfo' else 'RequesterVpcInfo']) - if 'PeeringOptions' in current_config['peer_info']: - current_config['peer_info'].pop('PeeringOptions') - if hasattr(self, 'organization') and current_config['peer_info']['OwnerId'] in self.organization: - current_config['peer_info']['name'] = self.organization[current_config['peer_info']['OwnerId']][ - 'Name'] - else: - current_config['peer_info']['name'] = current_config['peer_info']['OwnerId'] - def match_security_groups_and_resources_callback(self, current_config, path, current_path, resource_id, callback_args): service = current_path[1] diff --git a/ScoutSuite/providers/aws/services/vpc.py b/ScoutSuite/providers/aws/services/vpc.py deleted file mode 100644 index 2824350bb..000000000 --- a/ScoutSuite/providers/aws/services/vpc.py +++ /dev/null @@ -1,255 +0,0 @@ -# -*- coding: utf-8 -*- - -import copy - -import netaddr - -from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig -from ScoutSuite.providers.aws.configs.vpc import VPCConfig as SingleVPCConfig -from ScoutSuite.providers.base.configs.browser import get_value_at -from ScoutSuite.utils import manage_dictionary -from ScoutSuite.providers.aws.utils import ec2_classic, get_keys, get_name -from ScoutSuite.core.fs import load_data, read_ip_ranges - -######################################## -# Globals -######################################## - -protocols_dict = load_data('protocols.json', 'protocols') - - -######################################## -# VPCRegionConfig -######################################## - - -class VPCRegionConfig(RegionConfig): - """ - VPC configuration for a single AWS region - """ - customer_gateways = {} - flow_logs = {} - vpn_connections = {} - vpn_gateways = {} - - def parse_customer_gateway(self, global_params, region, cgw): - cgw['id'] = cgw.pop('CustomerGatewayId') - self.customer_gateways[cgw['id']] = cgw - - def parse_flow_log(self, global_params, region, flow_log): - """ - - :param global_params: - :param region: - :param flow_log: - :return: - """ - get_name(flow_log, flow_log, 'FlowLogId') - flow_log_id = flow_log.pop('FlowLogId') - self.flow_logs[flow_log_id] = flow_log - - def parse_network_acl(self, global_params, region, network_acl): - """ - - :param global_params: - :param region: - :param network_acl: - :return: - """ - vpc_id = network_acl['VpcId'] - network_acl['id'] = network_acl.pop('NetworkAclId') - get_name(network_acl, network_acl, 'id') - manage_dictionary(network_acl, 'rules', {}) - network_acl['rules']['ingress'] = self.__parse_network_acl_entries(network_acl['Entries'], False) - network_acl['rules']['egress'] = self.__parse_network_acl_entries(network_acl['Entries'], True) - network_acl.pop('Entries') - # Save - manage_dictionary(self.vpcs, vpc_id, SingleVPCConfig(self.vpc_resource_types)) - self.vpcs[vpc_id].network_acls[network_acl['id']] = network_acl - - @staticmethod - def __parse_network_acl_entries(entries, egress): - """ - - :param entries: - :param egress: - :return: - """ - acl_dict = {} - for entry in entries: - if entry['Egress'] == egress: - acl = {} - for key in ['RuleAction', 'RuleNumber']: - acl[key] = entry[key] - acl['CidrBlock'] = entry['CidrBlock'] if 'CidrBlock' in entry else entry['Ipv6CidrBlock'] - acl['protocol'] = protocols_dict[entry['Protocol']] - if 'PortRange' in entry: - from_port = entry['PortRange']['From'] if entry['PortRange']['From'] else 1 - to_port = entry['PortRange']['To'] if entry['PortRange']['To'] else 65535 - acl['port_range'] = from_port if from_port == to_port else str(from_port) + '-' + str(to_port) - else: - acl['port_range'] = '1-65535' - - acl_dict[acl.pop('RuleNumber')] = acl - return acl_dict - - def parse_route_table(self, global_params, region, rt): - route_table = {} - vpc_id = rt['VpcId'] - get_name(rt, route_table, 'VpcId') # TODO: change get_name to have src then dst - get_keys(rt, route_table, ['Routes', 'Associations', 'PropagatingVgws']) - # Save - manage_dictionary(self.vpcs, vpc_id, SingleVPCConfig(self.vpc_resource_types)) - self.vpcs[vpc_id].route_tables[rt['RouteTableId']] = route_table - - def parse_subnet(self, global_params, region, subnet): - """ - Parse subnet object. - - :param global_params: - :param region: - :param subnet: - :return: - """ - vpc_id = subnet['VpcId'] - manage_dictionary(self.vpcs, vpc_id, SingleVPCConfig(self.vpc_resource_types)) - subnet_id = subnet['SubnetId'] - get_name(subnet, subnet, 'SubnetId') - # set flow logs that cover this subnet - subnet['flow_logs'] = get_subnet_flow_logs_list(self, subnet) - # Save - manage_dictionary(self.vpcs, vpc_id, SingleVPCConfig(self.vpc_resource_types)) - self.vpcs[vpc_id].subnets[subnet_id] = subnet - - def parse_vpc(self, global_params, region_name, vpc): - """ - - :param global_params: - :param region_name: - :param vpc: - :return: - """ - vpc_id = vpc['VpcId'] - # Save - manage_dictionary(self.vpcs, vpc_id, SingleVPCConfig(self.vpc_resource_types)) - self.vpcs[vpc_id].name = get_name(vpc, {}, 'VpcId') - - def parse_vpn_connection(self, global_params, region_name, vpnc): - vpnc['id'] = vpnc.pop('VpnConnectionId') - self.vpn_connections[vpnc['id']] = vpnc - - def parse_vpn_gateway(self, global_params, region_name, vpng): - vpng['id'] = vpng.pop('VpnGatewayId') - self.vpn_gateways[vpng['id']] = vpng - - -######################################## -# VPCConfig -######################################## - - -class VPCConfig(RegionalServiceConfig): - """ - VPC configuration for all AWS regions - """ - - region_config_class = VPCRegionConfig - - def __init__(self, service_metadata, thread_config): - super(VPCConfig, self).__init__(service_metadata, thread_config) - - -######################################## -# VPC analysis functions -######################################## - -known_cidrs = {'0.0.0.0/0': 'All'} - - -def put_cidr_name(aws_config, current_config, path, current_path, resource_id, callback_args): - """ - Add a display name for all known CIDRs - :param aws_config: - :param current_config: - :param path: - :param current_path: - :param resource_id: - :param callback_args: - :return: - """ - if 'cidrs' in current_config: - cidr_list = [] - for cidr in current_config['cidrs']: - if type(cidr) == dict: - cidr = cidr['CIDR'] - if cidr in known_cidrs: - cidr_name = known_cidrs[cidr] - else: - cidr_name = get_cidr_name(cidr, callback_args['ip_ranges'], callback_args['ip_ranges_name_key']) - known_cidrs[cidr] = cidr_name - cidr_list.append({'CIDR': cidr, 'CIDRName': cidr_name}) - current_config['cidrs'] = cidr_list - - -aws_ip_ranges = {} # read_ip_ranges(aws_ip_ranges_filename, False) - - -def get_cidr_name(cidr, ip_ranges_files, ip_ranges_name_key): - """ - Read display name for CIDRs from ip-ranges files - :param cidr: - :param ip_ranges_files: - :param ip_ranges_name_key: - :return: - """ - for filename in ip_ranges_files: - ip_ranges = read_ip_ranges(filename, local_file=True) - for ip_range in ip_ranges: - ip_prefix = netaddr.IPNetwork(ip_range['ip_prefix']) - cidr = netaddr.IPNetwork(cidr) - if cidr in ip_prefix: - return ip_range[ip_ranges_name_key].strip() - for ip_range in aws_ip_ranges: - ip_prefix = netaddr.IPNetwork(ip_range['ip_prefix']) - cidr = netaddr.IPNetwork(cidr) - if cidr in ip_prefix: - return 'Unknown CIDR in %s %s' % (ip_range['service'], ip_range['region']) - return 'Unknown CIDR' - - -def propagate_vpc_names(aws_config, current_config, path, current_path, resource_id, callback_args): - """ - Propagate VPC names in VPC-related services (info only fetched during EC2 calls) - :param aws_config: - :param current_config: - :param path: - :param current_path: - :param resource_id: - :param callback_args: - :return: - """ - if resource_id == ec2_classic: - current_config['name'] = ec2_classic - else: - target_path = copy.deepcopy(current_path) - target_path[1] = 'ec2' - target_path.append(resource_id) - target_path.append('Name') - target_path = '.'.join(target_path) - current_config['name'] = get_value_at(aws_config, target_path, target_path) - - -def get_subnet_flow_logs_list(current_config, subnet): - """ - Return the flow logs that cover a given subnet - - :param current_config: - :param subnet: the subnet that the flow logs should cover - :return: - """ - flow_logs_list = [] - for flow_log in current_config.flow_logs: - if current_config.flow_logs[flow_log]['ResourceId'] == subnet['SubnetId'] or \ - current_config.flow_logs[flow_log]['ResourceId'] == subnet['VpcId']: - flow_logs_list.append(flow_log) - return flow_logs_list From bcbdc39aae6eec3a98608534938196618d78306a Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Thu, 21 Mar 2019 10:44:06 -0400 Subject: [PATCH 527/667] Integrated new DynamoDB implementation --- ScoutSuite/providers/aws/configs/services.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index b85678d78..86f6a10c7 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -24,11 +24,11 @@ try: from ScoutSuite.providers.aws.services.config_private import ConfigConfig - from ScoutSuite.providers.aws.services.dynamodb_private import DynamoDBConfig + from ScoutSuite.providers.aws.resources.dynamodb.service_private import DynamoDB from ScoutSuite.providers.aws.services.kms_private import KMSConfig except ImportError: ConfigConfig = None - DynamoDBConfig = None + DynamoDB = None KMSConfig = None @@ -79,8 +79,7 @@ def __init__(self, metadata=None, thread_config=4, **kwargs): try: self.config = ConfigConfig( metadata['management']['config'], thread_config) - self.dynamodb = DynamoDBConfig( - metadata['database']['dynamodb'], thread_config) + self.dynamodb = DynamoDB() self.kms = KMSConfig(metadata['security']['kms'], thread_config) except (NameError, TypeError): pass From d3214346cd6200a85d50fdeb76863e56a142827f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sgha=C3=AFer?= Date: Thu, 21 Mar 2019 19:21:28 -0400 Subject: [PATCH 528/667] Simplified code Co-Authored-By: Aboisier --- ScoutSuite/providers/aws/facade/s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/facade/s3.py b/ScoutSuite/providers/aws/facade/s3.py index 6c772f540..f01411341 100644 --- a/ScoutSuite/providers/aws/facade/s3.py +++ b/ScoutSuite/providers/aws/facade/s3.py @@ -13,7 +13,7 @@ class S3Facade(AWSBaseFacade): async def get_buckets(self): client = AWSFacadeUtils.get_client('s3', None, self.session) - buckets = (await run_concurrently(lambda: client.list_buckets()))['Buckets'] + buckets = (await run_concurrently(client.list_buckets))['Buckets'] for bucket in buckets: bucket_name = bucket['Name'] From 7755bacfb7c325256ec9753a40b6baf0288b75fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sgha=C3=AFer?= Date: Thu, 21 Mar 2019 19:21:47 -0400 Subject: [PATCH 529/667] Renamed Bucket to Buckets Co-Authored-By: Aboisier --- ScoutSuite/providers/aws/resources/s3/service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/s3/service.py b/ScoutSuite/providers/aws/resources/s3/service.py index f49e24dca..bba9e1081 100644 --- a/ScoutSuite/providers/aws/resources/s3/service.py +++ b/ScoutSuite/providers/aws/resources/s3/service.py @@ -5,7 +5,7 @@ from ScoutSuite.providers.utils import get_non_provider_id -class Bucket(AWSResources): +class Buckets(AWSResources): async def fetch_all(self, **kwargs): raw_buckets = await self.facade.s3.get_buckets() for raw_bucket in raw_buckets: @@ -38,7 +38,7 @@ def _parse_bucket(self, raw_bucket): class S3(AWSCompositeResources): _children = [ - (Bucket, 'buckets') + (Buckets, 'buckets') ] def __init__(self): From 7a639ee06c825f9dc213ed2fb7d4649a2bb43c18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sgha=C3=AFer?= Date: Thu, 21 Mar 2019 19:29:09 -0400 Subject: [PATCH 530/667] Parallelized a bit of stuff Co-Authored-By: Aboisier --- ScoutSuite/providers/aws/facade/iam.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/iam.py b/ScoutSuite/providers/aws/facade/iam.py index 1f09fd42e..9945e4907 100644 --- a/ScoutSuite/providers/aws/facade/iam.py +++ b/ScoutSuite/providers/aws/facade/iam.py @@ -3,18 +3,19 @@ from ScoutSuite.core.console import print_error, print_exception from ScoutSuite.providers.utils import run_concurrently, get_non_provider_id from ScoutSuite.providers.aws.utils import is_throttled +from ScoutSuite.providers.utils import run_concurrently class IAMFacade(AWSBaseFacade): async def get_credential_reports(self): client = AWSFacadeUtils.get_client('iam', self.session) - response = client.generate_credential_report() + response = await run_concurrently(client.generate_credential_report) if response['State'] != 'COMPLETE': print_error('Failed to generate a credential report.') return [] - report = client.get_credential_report()['Content'] + report = (await run_concurrently(client.get_credential_report))['Content'] # The report is a CSV string. The first row contains the name of each column. The next rows # each represent an individual account. This algorithm provides a simple initial parsing. From 21c7175302671bfe6cb285e26bc64a3bec3ae018 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sgha=C3=AFer?= Date: Thu, 21 Mar 2019 19:30:10 -0400 Subject: [PATCH 531/667] Parallelized more stuff Co-Authored-By: Aboisier --- ScoutSuite/providers/aws/facade/iam.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/iam.py b/ScoutSuite/providers/aws/facade/iam.py index 9945e4907..23d2d559e 100644 --- a/ScoutSuite/providers/aws/facade/iam.py +++ b/ScoutSuite/providers/aws/facade/iam.py @@ -50,8 +50,8 @@ async def get_policies(self): # TODO: Parallelize this for policy in policies: - policy_version = client.get_policy_version( - PolicyArn=policy['Arn'], VersionId=policy['DefaultVersionId']) + policy_version = await run_concurrently( + lambda: client.get_policy_version(PolicyArn=policy['Arn'], VersionId=policy['DefaultVersionId'])) policy['PolicyDocument'] = policy_version['PolicyVersion']['Document'] policy['attached_to'] = {} @@ -91,8 +91,8 @@ async def get_users(self): for group in groups: user['groups'].append(group['GroupName']) try: - user['LoginProfile'] = client.get_login_profile(UserName=user_name)[ - 'LoginProfile'] + user['LoginProfile'] = await run_concurrently( + lambda: client.get_login_profile(UserName=user_name)['LoginProfile']) except Exception: pass user['AccessKeys'] = await self._get_user_acces_keys(user_name) From 12327b6aad72fcc922df9af58b9a176bc1d757b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Sgha=C3=AFer?= Date: Thu, 21 Mar 2019 19:32:48 -0400 Subject: [PATCH 532/667] Apply suggestions from code review Co-Authored-By: Aboisier --- ScoutSuite/providers/aws/facade/iam.py | 3 +-- ScoutSuite/providers/aws/facade/utils.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/iam.py b/ScoutSuite/providers/aws/facade/iam.py index 23d2d559e..95420b71e 100644 --- a/ScoutSuite/providers/aws/facade/iam.py +++ b/ScoutSuite/providers/aws/facade/iam.py @@ -134,8 +134,7 @@ async def get_roles(self): async def get_password_policy(self): client = AWSFacadeUtils.get_client('iam', self.session) - response = await run_concurrently(lambda: client.get_account_password_policy()) - return response['PasswordPolicy'] + return (await run_concurrently(client.get_account_password_policy))['PasswordPolicy'] async def _get_user_acces_keys(self, user_name): client = AWSFacadeUtils.get_client('iam', self.session) diff --git a/ScoutSuite/providers/aws/facade/utils.py b/ScoutSuite/providers/aws/facade/utils.py index bb9ce31d1..3692db3e1 100644 --- a/ScoutSuite/providers/aws/facade/utils.py +++ b/ScoutSuite/providers/aws/facade/utils.py @@ -47,5 +47,5 @@ def get_client(service: str, session: boto3.session.Session, region: str=None): """ # TODO: investigate the use of a mutex to avoid useless creation of a same type of client among threads: - client = session.client(service, region_name=region) if region is not None else session.client(service) + client = session.client(service, region_name=region) if region else session.client(service) return AWSFacadeUtils._clients.setdefault((service, region), client) From 936784f3a01271a1561ad5063499527315b5c389 Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Fri, 22 Mar 2019 13:25:42 +0100 Subject: [PATCH 533/667] Set `` elements directly in the partial. --- ScoutSuite/output/data/html/partials/accordion_policy.html | 4 +++- .../aws/services.cloudformation.regions.id.stacks.html | 2 +- .../data/html/partials/aws/services.iam.inline_policies.html | 2 +- .../data/html/partials/aws/services.iam.managed_policies.html | 2 +- .../output/data/html/partials/aws/services.iam.roles.html | 2 +- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/ScoutSuite/output/data/html/partials/accordion_policy.html b/ScoutSuite/output/data/html/partials/accordion_policy.html index fbec5671a..79197a65f 100644 --- a/ScoutSuite/output/data/html/partials/accordion_policy.html +++ b/ScoutSuite/output/data/html/partials/accordion_policy.html @@ -8,7 +8,9 @@

    {{name}}

    - {{> policy}} + + {{> policy}} +
    diff --git a/ScoutSuite/output/data/html/partials/aws/services.cloudformation.regions.id.stacks.html b/ScoutSuite/output/data/html/partials/aws/services.cloudformation.regions.id.stacks.html index 93d540637..b3593ec74 100644 --- a/ScoutSuite/output/data/html/partials/aws/services.cloudformation.regions.id.stacks.html +++ b/ScoutSuite/output/data/html/partials/aws/services.cloudformation.regions.id.stacks.html @@ -35,7 +35,7 @@

    Capabilities {{> count_badge count=Capabilit {{#if policy}}
    - {{> accordion_policy name = 'Stack Policy' policy_path = (concat 'cloudformation.regions' region 'stacks' @key 'policy') document = policy}} + {{> accordion_policy name = 'Stack Policy' policy_path = (concat 'cloudformation.regions' region 'stacks' @key 'policy') document = policy}}
    {{/if}} diff --git a/ScoutSuite/output/data/html/partials/aws/services.iam.inline_policies.html b/ScoutSuite/output/data/html/partials/aws/services.iam.inline_policies.html index 9f72b2da2..cb4e531c7 100644 --- a/ScoutSuite/output/data/html/partials/aws/services.iam.inline_policies.html +++ b/ScoutSuite/output/data/html/partials/aws/services.iam.inline_policies.html @@ -7,7 +7,7 @@

    {{> count_badge count=inline_policies_count}}

    {{#each inline_policies}} - {{> accordion_policy name = name document = PolicyDocument policy_path = (concat 'iam' ../resource_type ../resource_id 'inline_policies' @key 'PolicyDocument')}} + {{> accordion_policy name = name document = PolicyDocument policy_path = (concat 'iam' ../resource_type ../resource_id 'inline_policies' @key 'PolicyDocument')}} {{/each}} diff --git a/ScoutSuite/output/data/html/partials/aws/services.iam.managed_policies.html b/ScoutSuite/output/data/html/partials/aws/services.iam.managed_policies.html index 12ceb3593..ef89641c9 100644 --- a/ScoutSuite/output/data/html/partials/aws/services.iam.managed_policies.html +++ b/ScoutSuite/output/data/html/partials/aws/services.iam.managed_policies.html @@ -6,7 +6,7 @@

    {{name}}

    Arn: {{arn}}
    - {{> accordion_policy document = PolicyDocument policy_path = (concat 'iam.policies' @key 'PolicyDocument')}} + {{> accordion_policy document = PolicyDocument policy_path = (concat 'iam.policies' @key 'PolicyDocument')}}<

    Attached entities

    diff --git a/ScoutSuite/output/data/html/partials/aws/services.iam.roles.html b/ScoutSuite/output/data/html/partials/aws/services.iam.roles.html index b073e61b7..077684e24 100644 --- a/ScoutSuite/output/data/html/partials/aws/services.iam.roles.html +++ b/ScoutSuite/output/data/html/partials/aws/services.iam.roles.html @@ -10,7 +10,7 @@

    Information

    ARN: {{arn}}
    - {{> accordion_policy name = 'Role Trust Policy' policy_path = (concat 'iam.roles' @key 'assume_role_policy.PolicyDocument') document = assume_role_policy.PolicyDocument}} + {{> accordion_policy name = 'Role Trust Policy' policy_path = (concat 'iam.roles' @key 'assume_role_policy.PolicyDocument') document = assume_role_policy.PolicyDocument}}
    From ac39a62e42ed9c7e1a3525ca1a32f77f37f6f18c Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Fri, 22 Mar 2019 13:28:38 +0100 Subject: [PATCH 534/667] Set `` to policy names. --- ScoutSuite/output/data/html/partials/accordion_policy.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/output/data/html/partials/accordion_policy.html b/ScoutSuite/output/data/html/partials/accordion_policy.html index 79197a65f..64e12f089 100644 --- a/ScoutSuite/output/data/html/partials/accordion_policy.html +++ b/ScoutSuite/output/data/html/partials/accordion_policy.html @@ -1,7 +1,7 @@ From b66f6b3fb57a44c99725b16dda8d021130618b5e Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Fri, 22 Mar 2019 16:34:06 +0100 Subject: [PATCH 536/667] Fix policy display --- .../output/data/html/partials/aws/services.iam.groups.html | 4 ++-- .../data/html/partials/aws/services.iam.inline_policies.html | 2 +- .../data/html/partials/aws/services.iam.managed_policies.html | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ScoutSuite/output/data/html/partials/aws/services.iam.groups.html b/ScoutSuite/output/data/html/partials/aws/services.iam.groups.html index 48eccaf32..d8657d831 100644 --- a/ScoutSuite/output/data/html/partials/aws/services.iam.groups.html +++ b/ScoutSuite/output/data/html/partials/aws/services.iam.groups.html @@ -11,12 +11,12 @@

    Information

    Members - {{> count_badge count=security_groups.length target=(concat '#iam.groups' id 'users')}} + {{> count_badge count=users.length target=(concat '#iam.groups' id 'users')}}

    diff --git a/ScoutSuite/output/data/html/partials/aws/services.iam.inline_policies.html b/ScoutSuite/output/data/html/partials/aws/services.iam.inline_policies.html index a226889cd..1f9168586 100644 --- a/ScoutSuite/output/data/html/partials/aws/services.iam.inline_policies.html +++ b/ScoutSuite/output/data/html/partials/aws/services.iam.inline_policies.html @@ -7,7 +7,7 @@

    {{> count_badge count=inline_policies_count}}

    {{#each inline_policies}} - {{> accordion_policy name=name document=PolicyDocument policy_path=(concat 'iam' ../resource_type ../resource_id 'inline_policies' @key 'PolicyDocument') heading="h5" samp=true}} + {{> accordion_policy name = name document = PolicyDocument policy_path = (concat 'iam' ../resource_type ../resource_id 'inline_policies' @key 'PolicyDocument') heading="h5" samp=true}} {{/each}}
    diff --git a/ScoutSuite/output/data/html/partials/aws/services.iam.managed_policies.html b/ScoutSuite/output/data/html/partials/aws/services.iam.managed_policies.html index ef89641c9..619d19a4c 100644 --- a/ScoutSuite/output/data/html/partials/aws/services.iam.managed_policies.html +++ b/ScoutSuite/output/data/html/partials/aws/services.iam.managed_policies.html @@ -5,8 +5,8 @@

    {{name}}

    -
    Arn: {{arn}}
    - {{> accordion_policy document = PolicyDocument policy_path = (concat 'iam.policies' @key 'PolicyDocument')}}< + + {{> accordion_policy name = arn document = PolicyDocument policy_path = (concat 'iam.policies' @key 'PolicyDocument') heading = "h5" }}

    Attached entities

    From 982d4199c0835b3e8b94053e5d4ffe263b8541a7 Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Fri, 22 Mar 2019 16:57:12 +0100 Subject: [PATCH 537/667] Fix policy display --- .../data/html/partials/aws/services.iam.managed_policies.html | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ScoutSuite/output/data/html/partials/aws/services.iam.managed_policies.html b/ScoutSuite/output/data/html/partials/aws/services.iam.managed_policies.html index 619d19a4c..202a119ee 100644 --- a/ScoutSuite/output/data/html/partials/aws/services.iam.managed_policies.html +++ b/ScoutSuite/output/data/html/partials/aws/services.iam.managed_policies.html @@ -5,11 +5,10 @@

    {{name}}

    - {{> accordion_policy name = arn document = PolicyDocument policy_path = (concat 'iam.policies' @key 'PolicyDocument') heading = "h5" }}
    -

    Attached entities

    +

    Attached Entities

      {{#each attached_to}}
    • {{make_title @key}}
      • From 24cedf6107b44983dfe7a20475696467253fcccd Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Fri, 22 Mar 2019 16:57:34 +0100 Subject: [PATCH 538/667] Resolve https://github.com/nccgroup/ScoutSuite/issues/253 --- .../html/partials/aws/services.iam.groups.html | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/ScoutSuite/output/data/html/partials/aws/services.iam.groups.html b/ScoutSuite/output/data/html/partials/aws/services.iam.groups.html index d8657d831..d5c51972b 100644 --- a/ScoutSuite/output/data/html/partials/aws/services.iam.groups.html +++ b/ScoutSuite/output/data/html/partials/aws/services.iam.groups.html @@ -9,20 +9,16 @@

      Information

      Creation date: {{CreateDate}}
    -
    -

    Members - {{> count_badge count=users.length target=(concat '#iam.groups' id 'users')}} +

    Members + {{> count_badge count=users.length target=(concat '#iam.groups' id 'users')}}

    -
    -
    -
    -
    {{> services.iam.inline_policies resource_type = 'groups' resource_id = @key}} {{> services.iam.policies_list}} From 3c233068391bfab1bc8ef7e9e763722421d1edeb Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Fri, 22 Mar 2019 17:08:27 +0100 Subject: [PATCH 539/667] Resolve https://github.com/nccgroup/ScoutSuite/issues/256 --- .../html/partials/aws/services.iam.managed_policies_list.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/output/data/html/partials/aws/services.iam.managed_policies_list.html b/ScoutSuite/output/data/html/partials/aws/services.iam.managed_policies_list.html index cbb1d34b7..de4fa0b88 100644 --- a/ScoutSuite/output/data/html/partials/aws/services.iam.managed_policies_list.html +++ b/ScoutSuite/output/data/html/partials/aws/services.iam.managed_policies_list.html @@ -2,7 +2,7 @@ From 0008e6f2b9540ddb44ed065e2aec1a9d17abeec6 Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Fri, 22 Mar 2019 17:17:27 +0100 Subject: [PATCH 541/667] Make badged not buttons --- ScoutSuite/output/data/html/partials/count_badge.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/output/data/html/partials/count_badge.html b/ScoutSuite/output/data/html/partials/count_badge.html index 9072b057f..62622fa5b 100644 --- a/ScoutSuite/output/data/html/partials/count_badge.html +++ b/ScoutSuite/output/data/html/partials/count_badge.html @@ -1,6 +1,6 @@ @@ -49,4 +64,3 @@

    User data

    - From 0b71279ff93c4ec83488f088dceaa3008808d7c0 Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Fri, 22 Mar 2019 19:03:40 +0100 Subject: [PATCH 545/667] Fix partial as described in https://github.com/nccgroup/ScoutSuite/issues/247 --- ...ices.ec2.regions.id.vpcs.id.instances.html | 10 +-- .../data/html/partials/network_interface.html | 64 +++++++++---------- 2 files changed, 35 insertions(+), 39 deletions(-) diff --git a/ScoutSuite/output/data/html/partials/aws/services.ec2.regions.id.vpcs.id.instances.html b/ScoutSuite/output/data/html/partials/aws/services.ec2.regions.id.vpcs.id.instances.html index 717cd184f..6dd2dc3a4 100644 --- a/ScoutSuite/output/data/html/partials/aws/services.ec2.regions.id.vpcs.id.instances.html +++ b/ScoutSuite/output/data/html/partials/aws/services.ec2.regions.id.vpcs.id.instances.html @@ -21,13 +21,9 @@

    Information

    Network interfaces

    -
      - {{#each network_interfaces}} -
    • - {{> network_interface region = ../region vpc = ../vpc network_interface = @../key}} -
      • - {{/each}} -
    + {{#each network_interfaces}} + {{> network_interface region = ../region vpc = ../vpc network_interface = @../key}} + {{/each}}
    {{#if user_data}}
    diff --git a/ScoutSuite/output/data/html/partials/network_interface.html b/ScoutSuite/output/data/html/partials/network_interface.html index c91700ebc..cc911006d 100644 --- a/ScoutSuite/output/data/html/partials/network_interface.html +++ b/ScoutSuite/output/data/html/partials/network_interface.html @@ -1,40 +1,40 @@ - From 62243e5dd0af5ca5c8f8944255c60242b4e8a3b3 Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Fri, 22 Mar 2019 19:20:13 +0100 Subject: [PATCH 546/667] Fix collapsibles --- .../services.elb.regions.id.vpcs.id.elbs.html | 2 +- ...ions.id.vpcs.id.elbs.linked_resources.html | 2 +- ...services.elbv2.regions.id.vpcs.id.lbs.html | 2 +- .../html/partials/aws/services.iam.roles.html | 2 +- .../html/partials/aws/services.s3.acls.html | 58 ++++++++++--------- .../aws/services.s3.bucket_iam_policies.html | 9 +-- .../partials/aws/services.s3.buckets.html | 2 +- .../aws/services.vpc.regions.id.vpcs.html | 4 +- 8 files changed, 41 insertions(+), 40 deletions(-) diff --git a/ScoutSuite/output/data/html/partials/aws/services.elb.regions.id.vpcs.id.elbs.html b/ScoutSuite/output/data/html/partials/aws/services.elb.regions.id.vpcs.id.elbs.html index dc2a825e8..26d0763de 100644 --- a/ScoutSuite/output/data/html/partials/aws/services.elb.regions.id.vpcs.id.elbs.html +++ b/ScoutSuite/output/data/html/partials/aws/services.elb.regions.id.vpcs.id.elbs.html @@ -47,7 +47,7 @@

    Attributes

    Security groups {{> count_badge count=security_groups.length target=(concat '#services.elb.regions' region 'vpcs' vpc 'elbs' @key 'security_groups')}}

    -
    +
      {{#each security_groups}} diff --git a/ScoutSuite/output/data/html/partials/aws/services.elb.regions.id.vpcs.id.elbs.linked_resources.html b/ScoutSuite/output/data/html/partials/aws/services.elb.regions.id.vpcs.id.elbs.linked_resources.html index 593e60cc4..ec0a1619a 100644 --- a/ScoutSuite/output/data/html/partials/aws/services.elb.regions.id.vpcs.id.elbs.linked_resources.html +++ b/ScoutSuite/output/data/html/partials/aws/services.elb.regions.id.vpcs.id.elbs.linked_resources.html @@ -8,7 +8,7 @@
      {{make_title resource_type {{> count_badge count=resources.length target=(concat '#services.elb.regions' region 'vpcs' vpc 'elbs' @key resource_type)}}
      -
      +
        {{#each resources}} diff --git a/ScoutSuite/output/data/html/partials/aws/services.elbv2.regions.id.vpcs.id.lbs.html b/ScoutSuite/output/data/html/partials/aws/services.elbv2.regions.id.vpcs.id.lbs.html index f5b6ff40a..02784cba6 100644 --- a/ScoutSuite/output/data/html/partials/aws/services.elbv2.regions.id.vpcs.id.lbs.html +++ b/ScoutSuite/output/data/html/partials/aws/services.elbv2.regions.id.vpcs.id.lbs.html @@ -44,7 +44,7 @@

        Attributes

        Security groups {{> count_badge count=security_groups.length target=(concat '#services.elbv2.regions' region 'vpcs' vpc 'lbs' @key 'security_groups')}}

        -
        +
          {{#each security_groups}} diff --git a/ScoutSuite/output/data/html/partials/aws/services.iam.roles.html b/ScoutSuite/output/data/html/partials/aws/services.iam.roles.html index 077684e24..99dafa4b7 100644 --- a/ScoutSuite/output/data/html/partials/aws/services.iam.roles.html +++ b/ScoutSuite/output/data/html/partials/aws/services.iam.roles.html @@ -17,7 +17,7 @@

          Information

          Instances {{> count_badge count=instances_count target=(concat '#iam.roles' id 'instances')}}

          -
          +
            {{#each instance_profiles}} diff --git a/ScoutSuite/output/data/html/partials/aws/services.s3.acls.html b/ScoutSuite/output/data/html/partials/aws/services.s3.acls.html index 95e8e8cbe..5d7d9c1a7 100644 --- a/ScoutSuite/output/data/html/partials/aws/services.s3.acls.html +++ b/ScoutSuite/output/data/html/partials/aws/services.s3.acls.html @@ -1,33 +1,39 @@ diff --git a/ScoutSuite/output/data/html/partials/aws/services.ec2.regions.vpcs.security_groups.resource_list.html b/ScoutSuite/output/data/html/partials/aws/services.ec2.regions.vpcs.security_groups.resource_list.html index 287dfd3a7..d79a451e7 100644 --- a/ScoutSuite/output/data/html/partials/aws/services.ec2.regions.vpcs.security_groups.resource_list.html +++ b/ScoutSuite/output/data/html/partials/aws/services.ec2.regions.vpcs.security_groups.resource_list.html @@ -4,9 +4,7 @@
          • {{ make_title service }} {{ make_title resource_type }} - - {{> count_badge count=resources.length}} - +
            • @@ -15,7 +13,8 @@
            {{ make_title service }} { {{#each resources}}
          • - {{get_value_at 'services' ../service 'regions' ../region 'vpcs' ../vpc ../resource_type this 'name'}} + {{this}} +
            • {{/each}} diff --git a/ScoutSuite/output/data/html/partials/aws/services.elb.regions.id.vpcs.id.elbs.html b/ScoutSuite/output/data/html/partials/aws/services.elb.regions.id.vpcs.id.elbs.html index 26d0763de..80270e6bb 100644 --- a/ScoutSuite/output/data/html/partials/aws/services.elb.regions.id.vpcs.id.elbs.html +++ b/ScoutSuite/output/data/html/partials/aws/services.elb.regions.id.vpcs.id.elbs.html @@ -23,11 +23,7 @@

            Listeners

              {{#each listeners}}
            • - {{> accordion region = ../region items = policies - title_partial = 'services.elb.regions.vpcs.elbs.listener' - item_partial = 'services.elb.regions.vpcs.elbs.linked_policy' - accordion_id = (concat 'services.elb.regions' ../region 'vpcs' ../vpc 'elbs' @../key) - }} + {{@key}} ({{Protocol}}{{#if SslPolicy}}, {{SslPolicy}}{{/if}})
              • {{/each}}
            @@ -70,3 +66,12 @@

            Destination

            Handlebars.registerPartial("services.elb.regions.id.vpcs.id.elbs", $("#services\\.elb\\.regions\\.id\\.vpcs\\.id\\.elbs\\.partial").html()); + + + + + diff --git a/ScoutSuite/output/data/html/partials/aws/services.elbv2.regions.id.vpcs.id.lbs.html b/ScoutSuite/output/data/html/partials/aws/services.elbv2.regions.id.vpcs.id.lbs.html index 02784cba6..809730588 100644 --- a/ScoutSuite/output/data/html/partials/aws/services.elbv2.regions.id.vpcs.id.lbs.html +++ b/ScoutSuite/output/data/html/partials/aws/services.elbv2.regions.id.vpcs.id.lbs.html @@ -61,3 +61,11 @@

            Security groups Handlebars.registerPartial("services.elbv2.regions.id.vpcs.id.lbs", $("#services\\.elbv2\\.regions\\.id\\.vpcs\\.id\\.lbs\\.partial").html()); + + + + diff --git a/ScoutSuite/output/data/html/partials/network_interface.html b/ScoutSuite/output/data/html/partials/network_interface.html index cc911006d..f9c64137d 100644 --- a/ScoutSuite/output/data/html/partials/network_interface.html +++ b/ScoutSuite/output/data/html/partials/network_interface.html @@ -42,7 +42,7 @@

            {{@key}} Handlebars.registerPartial("network_interface", $("#network_interface\\.partial").html()); - + From 52d4fafca2ecc1950e7c11caa0760c022a7fbeb3 Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Sat, 23 Mar 2019 19:11:44 +0100 Subject: [PATCH 548/667] Fix for https://github.com/nccgroup/ScoutSuite/issues/104 --- ScoutSuite/providers/base/provider.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ScoutSuite/providers/base/provider.py b/ScoutSuite/providers/base/provider.py index 259b258cc..11af7dfc0 100644 --- a/ScoutSuite/providers/base/provider.py +++ b/ScoutSuite/providers/base/provider.py @@ -111,6 +111,16 @@ def _load_metadata(self): @staticmethod def _build_services_list(supported_services, services, skipped_services): + + # Ensure services and skipped services exist, otherwise log exception + error = False + for service in services+skipped_services: + if service not in supported_services: + print_exception('Service \"{}\" does not exist, skipping.'.format(service)) + error = True + if error: + print_exception('Available services are: {}'.format(str(list(supported_services)).strip('[]'))) + return [s for s in supported_services if (services == [] or s in services) and s not in skipped_services] def _update_last_run(self, current_time, ruleset): From 22cade9e7fdaf22bff6960608dc5a6113214dad1 Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Sat, 23 Mar 2019 19:18:36 +0100 Subject: [PATCH 549/667] Change title --- ScoutSuite/output/data/html/partials/last_run_details.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/output/data/html/partials/last_run_details.html b/ScoutSuite/output/data/html/partials/last_run_details.html index c8cd7786c..d4aa0f752 100644 --- a/ScoutSuite/output/data/html/partials/last_run_details.html +++ b/ScoutSuite/output/data/html/partials/last_run_details.html @@ -2,7 +2,7 @@
            -
            Last run details
            +
            Execution Details
            From 9f0d5fd16664f8b2324df13249e61acf1c30f254 Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Sat, 23 Mar 2019 19:27:26 +0100 Subject: [PATCH 550/667] Cosmetic changes to modals --- ScoutSuite/output/data/html/partials/about_scoutsuite.html | 6 +++--- ScoutSuite/output/data/html/partials/metadata.html | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ScoutSuite/output/data/html/partials/about_scoutsuite.html b/ScoutSuite/output/data/html/partials/about_scoutsuite.html index 78f72dcc0..21ac2781a 100644 --- a/ScoutSuite/output/data/html/partials/about_scoutsuite.html +++ b/ScoutSuite/output/data/html/partials/about_scoutsuite.html @@ -1,5 +1,5 @@ + + diff --git a/ScoutSuite/output/data/inc-scoutsuite/scoutsuite.js b/ScoutSuite/output/data/inc-scoutsuite/scoutsuite.js index 1ac1a0165..8e0bfc579 100644 --- a/ScoutSuite/output/data/inc-scoutsuite/scoutsuite.js +++ b/ScoutSuite/output/data/inc-scoutsuite/scoutsuite.js @@ -71,7 +71,6 @@ function onPageLoad() { download_as_csv(finding_key + '.csv', csv_array) }; - ; if (button_clicked == 'findings_download_json_button') { json_dict['items'] = []; @@ -768,6 +767,13 @@ function showLastRunDetails() { $('#modal-container').modal(); } +/** + * Shows resources details modal + */ +function showResourcesDetails() { + $('#modal-container').html(resources_details_template(run_results)); + $('#modal-container').modal(); +} /** * Show main dashboard */ From 60e2f05c4c485ff62abe243da02b71730d2b6332 Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Sun, 24 Mar 2019 12:23:17 +0100 Subject: [PATCH 552/667] Hide resources with no count --- ScoutSuite/output/data/html/partials/resources_details.html | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ScoutSuite/output/data/html/partials/resources_details.html b/ScoutSuite/output/data/html/partials/resources_details.html index 517b2b40f..3f119cdfa 100644 --- a/ScoutSuite/output/data/html/partials/resources_details.html +++ b/ScoutSuite/output/data/html/partials/resources_details.html @@ -22,12 +22,10 @@
            Resources Details
            {{#each this}} {{#each this.resources}} - {{make_title @../key}} - {{make_title @key}} {{#if this.count}} + {{make_title @../key}} + {{make_title @key}} {{this.count}} - {{else}} - 0 {{/if}} {{/each}} From 3a82991d20352fd98e6540d9e9f268deecf1d368 Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Sun, 24 Mar 2019 13:36:24 +0100 Subject: [PATCH 553/667] Partial refactoring to use coloredlogs --- ScoutSuite/__main__.py | 4 +-- ScoutSuite/core/console.py | 57 ++++++++++++++++++++++---------------- ScoutSuite/core/fs.py | 7 ++--- ScoutSuite/output/js.py | 6 ++-- requirements.txt | 1 + 5 files changed, 42 insertions(+), 33 deletions(-) diff --git a/ScoutSuite/__main__.py b/ScoutSuite/__main__.py index 00eaec87e..1ac6c9e2f 100644 --- a/ScoutSuite/__main__.py +++ b/ScoutSuite/__main__.py @@ -7,7 +7,7 @@ from ScoutSuite import AWSCONFIG from ScoutSuite.core.cli_parser import ScoutSuiteArgumentParser -from ScoutSuite.core.console import config_debug_level, print_info, print_debug +from ScoutSuite.core.console import set_config_debug_level, print_info, print_debug from ScoutSuite.core.exceptions import RuleExceptions from ScoutSuite.core.processingengine import ProcessingEngine from ScoutSuite.core.ruleset import Ruleset @@ -30,7 +30,7 @@ async def main(args=None): args = args.__dict__ # Configure the debug level - config_debug_level(args.get('debug')) + set_config_debug_level(args.get('debug')) # Create a cloud provider object cloud_provider = get_provider(provider=args.get('provider'), diff --git a/ScoutSuite/core/console.py b/ScoutSuite/core/console.py index 95a0d2346..72deacd14 100644 --- a/ScoutSuite/core/console.py +++ b/ScoutSuite/core/console.py @@ -4,53 +4,59 @@ from six.moves import input +import coloredlogs, logging ######################################## -# Print configuration functions +# Output configuration ######################################## -def config_debug_level(enable): - """ - Configure whether full stacktraces should be dumped in the console output +verbose_exceptions = False +logger = logging.getLogger('scout') - :param enable: +errors_list = [] +exceptions_list = [] - :return: +def set_config_debug_level(is_debug): + """ + Configure whether full stacktraces should be dumped in the console output """ global verbose_exceptions - verbose_exceptions = enable + verbose_exceptions = is_debug + coloredlogs.install(level='DEBUG' if is_debug else 'INFO') ######################################## -# Print functions +# Output functions ######################################## +def print_generic(msg, new_line=True): + if new_line: + msg = '{}\n'.format(msg) + logger.info(msg) + + def print_debug(msg): - if verbose_exceptions: - print_generic(sys.stderr, msg) + logger.debug(msg) def print_error(msg, new_line=True): - print_generic(sys.stderr, msg, new_line) + if new_line: + msg = '{}\n'.format(msg) + logger.error(msg) def print_exception(e, debug_only=False): global verbose_exceptions + if verbose_exceptions: - print_error(str(traceback.format_exc())) + logger.error(traceback.format_exc()) elif not debug_only: - print_error(str(e)) - - -def print_generic(out, msg, new_line=True): - out.write(msg) - out.flush() - if new_line: - out.write('\n') + logger.error(e) def print_info(msg, new_line=True): - print_generic(sys.stdout, msg, new_line) + + print_generic(msg, new_line) ######################################## @@ -94,7 +100,8 @@ def prompt_overwrite(filename, force_write, test_input=None): def prompt_value(question, choices=None, default=None, display_choices=True, display_indices=False, - authorize_list=False, is_question=False, no_confirm=False, required=True, regex=None, + authorize_list=False, is_question=False, no_confirm=False, required=True, + regex=None, regex_format='', max_laps=5, test_input=None, return_index=False): """ Prompt for a value @@ -152,7 +159,8 @@ def prompt_value(question, choices=None, default=None, display_choices=True, dis elif choices: user_choices = [item.strip() for item in choice.split(',')] if not authorize_list and len(user_choices) > 1: - print_error('Error: multiple values are not supported; please enter a single value.') + print_error( + 'Error: multiple values are not supported; please enter a single value.') else: choice_valid = True if display_indices and int(choice) < len(choices): @@ -178,7 +186,8 @@ def prompt_value(question, choices=None, default=None, display_choices=True, dis can_return = True if can_return: # Manually confirm that the entered value is correct if needed - if no_confirm or prompt_yes_no('You entered "' + choice + '". Is that correct', test_input=test_input): + if no_confirm or prompt_yes_no('You entered "' + choice + '". Is that correct', + test_input=test_input): return int(int_choice) if return_index else choice diff --git a/ScoutSuite/core/fs.py b/ScoutSuite/core/fs.py index bfba11aaa..e176263df 100644 --- a/ScoutSuite/core/fs.py +++ b/ScoutSuite/core/fs.py @@ -5,7 +5,7 @@ import json import os -from ScoutSuite.core.console import print_exception, prompt_overwrite +from ScoutSuite.core.console import print_exception, prompt_overwrite, print_info from ScoutSuite.core.conditions import pass_condition @@ -101,8 +101,7 @@ def save_blob_as_json(filename, blob, force_write): try: if prompt_overwrite(filename, force_write): with open(filename, 'wt') as f: - print('%s' % json.dumps(blob, indent=4, separators=(',', ': '), sort_keys=True, - cls=CustomJSONEncoder), file=f) + print_info('%s' % json.dumps(blob, indent=4, separators=(',', ': '), sort_keys=True, + cls=CustomJSONEncoder), file=f) except Exception as e: print_exception(e) - pass diff --git a/ScoutSuite/output/js.py b/ScoutSuite/output/js.py index d0b56fee3..e91df64db 100644 --- a/ScoutSuite/output/js.py +++ b/ScoutSuite/output/js.py @@ -63,12 +63,12 @@ def load_from_file(self, config_type, config_path = None, first_line = None): def save_to_file(self, config, config_type, force_write, debug): config_path, first_line = get_filename(config_type, self.profile, self.report_dir) - print('Saving data to %s' % config_path) + print_info('Saving data to %s' % config_path) try: with self.__open_file(config_path, force_write, False) as f: if first_line: - print('%s' % first_line, file=f) - print('%s' % json.dumps(config, indent=4 if debug else None, separators=(',', ': '), sort_keys=True, cls=Scout2Encoder), file=f) + print_info('%s' % first_line, file=f) + print_info('%s' % json.dumps(config, indent=4 if debug else None, separators=(',', ': '), sort_keys=True, cls=Scout2Encoder), file=f) except Exception as e: print_exception(e) diff --git a/requirements.txt b/requirements.txt index 9b42a454f..2d419ee38 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ python-dateutil>=2.2 netaddr>=0.7.11 iampoliciesgonewild>=1.0.6.2 +coloredlogs>=10.0 # AWS Provider boto3>=1.9.60 From e3cc5dd82f2ee88a5a3b891bb32b90401803c621 Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Sun, 24 Mar 2019 14:17:21 +0100 Subject: [PATCH 554/667] Partial refactoring to use coloredlogs --- ScoutSuite/core/console.py | 14 +++++--------- ScoutSuite/output/js.py | 4 ++-- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/ScoutSuite/core/console.py b/ScoutSuite/core/console.py index 72deacd14..bf529d08a 100644 --- a/ScoutSuite/core/console.py +++ b/ScoutSuite/core/console.py @@ -22,16 +22,14 @@ def set_config_debug_level(is_debug): """ global verbose_exceptions verbose_exceptions = is_debug - coloredlogs.install(level='DEBUG' if is_debug else 'INFO') + coloredlogs.install(level='DEBUG' if is_debug else 'INFO', logger=logger) ######################################## # Output functions ######################################## -def print_generic(msg, new_line=True): - if new_line: - msg = '{}\n'.format(msg) +def print_generic(msg): logger.info(msg) @@ -39,9 +37,7 @@ def print_debug(msg): logger.debug(msg) -def print_error(msg, new_line=True): - if new_line: - msg = '{}\n'.format(msg) +def print_error(msg): logger.error(msg) @@ -54,9 +50,9 @@ def print_exception(e, debug_only=False): logger.error(e) -def print_info(msg, new_line=True): +def print_info(msg): - print_generic(msg, new_line) + print_generic(msg) ######################################## diff --git a/ScoutSuite/output/js.py b/ScoutSuite/output/js.py index e91df64db..b743a42d7 100644 --- a/ScoutSuite/output/js.py +++ b/ScoutSuite/output/js.py @@ -67,8 +67,8 @@ def save_to_file(self, config, config_type, force_write, debug): try: with self.__open_file(config_path, force_write, False) as f: if first_line: - print_info('%s' % first_line, file=f) - print_info('%s' % json.dumps(config, indent=4 if debug else None, separators=(',', ': '), sort_keys=True, cls=Scout2Encoder), file=f) + print_info('%s' % first_line) + print_info('%s' % json.dumps(config, indent=4 if debug else None, separators=(',', ': '), sort_keys=True, cls=Scout2Encoder)) except Exception as e: print_exception(e) From 47f2c02e05c73f8364f6c161d915008c7e9ca63c Mon Sep 17 00:00:00 2001 From: misg Date: Sun, 24 Mar 2019 12:14:54 -0400 Subject: [PATCH 555/667] Add private KMS facade to AWS facade. --- ScoutSuite/providers/aws/facade/facade.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/ScoutSuite/providers/aws/facade/facade.py b/ScoutSuite/providers/aws/facade/facade.py index 9d0649c20..ce3abc00d 100644 --- a/ScoutSuite/providers/aws/facade/facade.py +++ b/ScoutSuite/providers/aws/facade/facade.py @@ -16,6 +16,11 @@ from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade from ScoutSuite.providers.utils import run_concurrently +try: + from ScoutSuite.providers.aws.facade.kms_private import KMSFacade +except NameError: + pass + class AWSFacade(AWSBaseFacade): def __init__(self, credentials: dict = None): @@ -33,6 +38,11 @@ def __init__(self, credentials: dict = None): self.elbv2 = ELBv2Facade(self.session) self.redshift = RedshiftFacade(self.session) + try: + self.kms = KMSFacade(self.session) + except NameError: + pass + async def build_region_list(self, service: str, chosen_regions=None, partition_name='aws'): service = 'ec2containerservice' if service == 'ecs' else service available_services = await run_concurrently(lambda: Session().get_available_services()) @@ -72,3 +82,8 @@ def _set_session(self, credentials: dict): self.emr = EMRFacade(self.session) self.elbv2 = ELBv2Facade(self.session) self.redshift = RedshiftFacade(self.session) + + try: + self.kms = KMSFacade(self.session) + except NameError: + pass From ed7736fc63ffb96eb61e1fccdbf06bd96d2f9e8b Mon Sep 17 00:00:00 2001 From: misg Date: Sun, 24 Mar 2019 12:15:21 -0400 Subject: [PATCH 556/667] Migrate KMS service to the new architecture. --- ScoutSuite/providers/aws/configs/services.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index b85678d78..555631e18 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -25,11 +25,11 @@ try: from ScoutSuite.providers.aws.services.config_private import ConfigConfig from ScoutSuite.providers.aws.services.dynamodb_private import DynamoDBConfig - from ScoutSuite.providers.aws.services.kms_private import KMSConfig + from ScoutSuite.providers.aws.resources.kms.service_private import KMS except ImportError: ConfigConfig = None DynamoDBConfig = None - KMSConfig = None + KMS = None class AWSServicesConfig(BaseServicesConfig): @@ -81,7 +81,7 @@ def __init__(self, metadata=None, thread_config=4, **kwargs): metadata['management']['config'], thread_config) self.dynamodb = DynamoDBConfig( metadata['database']['dynamodb'], thread_config) - self.kms = KMSConfig(metadata['security']['kms'], thread_config) + self.kms = KMS() except (NameError, TypeError): pass From b409bb398218c7ca38337cccf02a31837ec154a3 Mon Sep 17 00:00:00 2001 From: misg Date: Sun, 24 Mar 2019 12:15:39 -0400 Subject: [PATCH 557/667] Remove old KMS service support. --- ScoutSuite/providers/aws/metadata.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/ScoutSuite/providers/aws/metadata.json b/ScoutSuite/providers/aws/metadata.json index fdd424467..2e4cfebef 100644 --- a/ScoutSuite/providers/aws/metadata.json +++ b/ScoutSuite/providers/aws/metadata.json @@ -368,8 +368,6 @@ "kms": { "resources": { "keys": { - "api_call": "list_keys", - "response": "Keys", "path": "services.kms.regions.id.keys" } } From c5b3a3a03bc47b1c1d38acc0e634d23e9d637a4f Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Sun, 24 Mar 2019 12:43:46 -0400 Subject: [PATCH 558/667] Fixed S3 Bucket ACLs table spill --- .../html/partials/aws/services.s3.acls.html | 14 ++++++------- .../data/inc-scoutsuite/css/scoutsuite.css | 20 ++++++++++++++++++- .../output/data/inc-scoutsuite/scoutsuite.js | 11 ++++++++++ 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/ScoutSuite/output/data/html/partials/aws/services.s3.acls.html b/ScoutSuite/output/data/html/partials/aws/services.s3.acls.html index 5d7d9c1a7..0fbb87ac7 100644 --- a/ScoutSuite/output/data/html/partials/aws/services.s3.acls.html +++ b/ScoutSuite/output/data/html/partials/aws/services.s3.acls.html @@ -6,16 +6,16 @@

            {{make_title resource_type}} ACLs

            - - - - - - + + + + + + {{#each grantees}} - + diff --git a/ScoutSuite/output/data/inc-scoutsuite/css/scoutsuite.css b/ScoutSuite/output/data/inc-scoutsuite/css/scoutsuite.css index d2980c459..4972b00a6 100644 --- a/ScoutSuite/output/data/inc-scoutsuite/css/scoutsuite.css +++ b/ScoutSuite/output/data/inc-scoutsuite/css/scoutsuite.css @@ -38,7 +38,6 @@ /* Scout Suite */ - .item-margin { margin-left: 25px; } @@ -328,6 +327,25 @@ input:checked + .slider:before { font-size: 18px; } +.list-group-item { + overflow: auto; +} + +.bucket-name { + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + display: block; + width: 150px; +} +.bucket-name:hover { + cursor: pointer; +} + +.table-padding > * { + padding: 10px; +} + @media only screen and (max-width: 1200px) { #container-user { margin-top: 5rem !important; diff --git a/ScoutSuite/output/data/inc-scoutsuite/scoutsuite.js b/ScoutSuite/output/data/inc-scoutsuite/scoutsuite.js index 1ac1a0165..2abf0ef8d 100644 --- a/ScoutSuite/output/data/inc-scoutsuite/scoutsuite.js +++ b/ScoutSuite/output/data/inc-scoutsuite/scoutsuite.js @@ -1058,6 +1058,17 @@ function make_title(title) { }; }; +/** + * Toggles between truncated and full lenght bucket name + */ +function toggleName(name) { + if (name.style.display !== 'contents') { + name.style.display = 'contents' + } else { + name.style.display = 'block' + } +} + /** * Add one or * @param group From 24ce3734cb7d6e9cc78d6941578d90c4105cd519 Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Sun, 24 Mar 2019 19:18:40 +0100 Subject: [PATCH 559/667] Reverting breaking changes --- ScoutSuite/output/js.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ScoutSuite/output/js.py b/ScoutSuite/output/js.py index b743a42d7..7a118435d 100644 --- a/ScoutSuite/output/js.py +++ b/ScoutSuite/output/js.py @@ -66,9 +66,10 @@ def save_to_file(self, config, config_type, force_write, debug): print_info('Saving data to %s' % config_path) try: with self.__open_file(config_path, force_write, False) as f: + # TODO fix this if first_line: - print_info('%s' % first_line) - print_info('%s' % json.dumps(config, indent=4 if debug else None, separators=(',', ': '), sort_keys=True, cls=Scout2Encoder)) + print('%s' % first_line, file=f) + print('%s' % json.dumps(config, indent=4 if debug else None, separators=(',', ': '), sort_keys=True, cls=Scout2Encoder), file=f) except Exception as e: print_exception(e) @@ -96,4 +97,4 @@ def __open_file(self, config_filename, force_write, quiet=False): except Exception as e: print_exception(e) else: - return None + return None \ No newline at end of file From a9fd1563679d86d53c5bb9e456ac08b750062cf7 Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Sun, 24 Mar 2019 19:18:56 +0100 Subject: [PATCH 560/667] Adding output to __main__ --- ScoutSuite/__main__.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/__main__.py b/ScoutSuite/__main__.py index 1ac6c9e2f..23ddc5aee 100644 --- a/ScoutSuite/__main__.py +++ b/ScoutSuite/__main__.py @@ -7,7 +7,7 @@ from ScoutSuite import AWSCONFIG from ScoutSuite.core.cli_parser import ScoutSuiteArgumentParser -from ScoutSuite.core.console import set_config_debug_level, print_info, print_debug +from ScoutSuite.core.console import set_config_debug_level, print_info, print_debug, print_error from ScoutSuite.core.exceptions import RuleExceptions from ScoutSuite.core.processingengine import ProcessingEngine from ScoutSuite.core.ruleset import Ruleset @@ -33,6 +33,7 @@ async def main(args=None): set_config_debug_level(args.get('debug')) # Create a cloud provider object + print_info('Creating provider connector') cloud_provider = get_provider(provider=args.get('provider'), profile=args.get('profile'), project_id=args.get('project_id'), @@ -50,11 +51,13 @@ async def main(args=None): # TODO: move this to after authentication, so that the report can be more specific to what's being scanned. # For example if scanning with a GCP service account, the SA email can only be known after authenticating... # Create a new report + print_info('Initializing report') report = Scout2Report(args.get('provider'), report_file_name, args.get( 'report_dir'), args.get('timestamp')) # Complete run, including pulling data from provider if not args.get('fetch_local'): + print_info('Authenticating to cloud provider') # Authenticate to the cloud provider authenticated = cloud_provider.authenticate(profile=args.get('profile'), user_account=args.get('user_account'), @@ -71,10 +74,12 @@ async def main(args=None): password=args.get('password')) if not authenticated: + print_error("Authentication failure") return 401 # Fetch data from provider APIs try: + print_info('Fetching data from provider APIs') await cloud_provider.fetch(regions=args.get('regions')) except KeyboardInterrupt: print_info('\nCancelled by user') @@ -82,6 +87,7 @@ async def main(args=None): # Update means we reload the whole config and overwrite part of it if args.get('update'): + print_info('Updating existing data') current_run_services = copy.deepcopy(cloud_provider.services) last_run_dict = report.jsrw.load_from_file(AWSCONFIG) cloud_provider.services = last_run_dict['services'] @@ -90,6 +96,7 @@ async def main(args=None): # Partial run, using pre-pulled data else: + print_info('Using local data') # Reload to flatten everything into a python dictionary last_run_dict = report.jsrw.load_from_file(AWSCONFIG) for key in last_run_dict: @@ -100,6 +107,7 @@ async def main(args=None): args.get('ip_ranges'), args.get('ip_ranges_name_key')) # Analyze config + print_info('Running rule engine') finding_rules = Ruleset(environment_name=args.get('profile'), cloud_provider=args.get('provider'), filename=args.get('ruleset'), @@ -109,6 +117,7 @@ async def main(args=None): processing_engine.run(cloud_provider) # Create display filters + print_info('Applying display filters') filter_rules = Ruleset(cloud_provider=args.get('provider'), filename='filters.json', rule_type='filters', From c11c6ddc4bdd4d3bff1020784cf26f4b2962bb0c Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Sun, 24 Mar 2019 19:27:43 +0100 Subject: [PATCH 561/667] Remove TODO as it's not required (isn't really a print) --- ScoutSuite/output/js.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ScoutSuite/output/js.py b/ScoutSuite/output/js.py index 7a118435d..c56d40853 100644 --- a/ScoutSuite/output/js.py +++ b/ScoutSuite/output/js.py @@ -66,7 +66,6 @@ def save_to_file(self, config, config_type, force_write, debug): print_info('Saving data to %s' % config_path) try: with self.__open_file(config_path, force_write, False) as f: - # TODO fix this if first_line: print('%s' % first_line, file=f) print('%s' % json.dumps(config, indent=4 if debug else None, separators=(',', ': '), sort_keys=True, cls=Scout2Encoder), file=f) From 751c70f13fb0a5a53ee6f1e405ef42275970c14f Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Sun, 24 Mar 2019 19:57:50 +0100 Subject: [PATCH 562/667] Improvements --- ScoutSuite/__main__.py | 4 ++-- ScoutSuite/core/console.py | 17 +++++++++----- ScoutSuite/providers/base/configs/services.py | 3 ++- ScoutSuite/providers/base/provider.py | 22 ++++++++++--------- 4 files changed, 27 insertions(+), 19 deletions(-) diff --git a/ScoutSuite/__main__.py b/ScoutSuite/__main__.py index 23ddc5aee..8eb8eec56 100644 --- a/ScoutSuite/__main__.py +++ b/ScoutSuite/__main__.py @@ -33,7 +33,7 @@ async def main(args=None): set_config_debug_level(args.get('debug')) # Create a cloud provider object - print_info('Creating provider connector') + print_info('Creating {} connector'.format(args.get('provider'))) cloud_provider = get_provider(provider=args.get('provider'), profile=args.get('profile'), project_id=args.get('project_id'), @@ -57,7 +57,7 @@ async def main(args=None): # Complete run, including pulling data from provider if not args.get('fetch_local'): - print_info('Authenticating to cloud provider') + print_info('Authenticating to {}'.format(args.get('provider'))) # Authenticate to the cloud provider authenticated = cloud_provider.authenticate(profile=args.get('profile'), user_account=args.get('user_account'), diff --git a/ScoutSuite/core/console.py b/ScoutSuite/core/console.py index bf529d08a..429c1dc4d 100644 --- a/ScoutSuite/core/console.py +++ b/ScoutSuite/core/console.py @@ -12,6 +12,7 @@ verbose_exceptions = False logger = logging.getLogger('scout') +# TODO logger should name the file where the error occurred errors_list = [] exceptions_list = [] @@ -41,13 +42,17 @@ def print_error(msg): logger.error(msg) -def print_exception(e, debug_only=False): - global verbose_exceptions +def print_exception(exception, additional_details): + logger.error(exception) + + # TODO do something with these + global exceptions_list + exceptions_list.append({'exception': exception, + 'traceback': traceback.format_exc(), + 'additional_details': additional_details}) - if verbose_exceptions: - logger.error(traceback.format_exc()) - elif not debug_only: - logger.error(e) + a = exceptions_list + b = 1 def print_info(msg): diff --git a/ScoutSuite/providers/base/configs/services.py b/ScoutSuite/providers/base/configs/services.py index 1feea98f2..460c4a186 100644 --- a/ScoutSuite/providers/base/configs/services.py +++ b/ScoutSuite/providers/base/configs/services.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from ScoutSuite.core.console import print_error, print_exception, print_debug +from ScoutSuite.core.console import print_error, print_exception, print_debug, print_info from ScoutSuite.providers.aws.utils import get_partition_name @@ -20,6 +20,7 @@ async def fetch(self, credentials, services=None, regions=None): # skip services if services != [] and service not in services: continue + print_info('Fetching resources for the {} service'.format(service)) # TODO pretty name service_config = getattr(self, service) # call fetch method for the service if 'fetch_all' in dir(service_config): diff --git a/ScoutSuite/providers/base/provider.py b/ScoutSuite/providers/base/provider.py index 0b9e13b2f..73644d834 100644 --- a/ScoutSuite/providers/base/provider.py +++ b/ScoutSuite/providers/base/provider.py @@ -326,11 +326,12 @@ def _go_to_and_do(self, current_config, path, current_path, callback, callback_a callback_args) except Exception as e: - print_exception(e) - print_info('Path: %s' % str(current_path)) - print_info('Key = %s' % str(key) if 'key' in locals() else 'not defined') - print_info('Value = %s' % str(value) if 'value' in locals() else 'not defined') - print_info('Path = %s' % path) + print_exception(e, {'current path': '{}'.format(current_path), + 'key': '{}'.format(key if 'key' in locals() else 'not defined'), + 'value': '{}'.format(value if 'value' in locals() else 'not defined'), + 'path': '{}'.format(path), + } + ) def _new_go_to_and_do(self, current_config, path, current_path, callbacks): """ @@ -381,8 +382,9 @@ def _new_go_to_and_do(self, current_config, path, current_path, callbacks): tmp.append(i) self._new_go_to_and_do(current_config[key][i], copy.deepcopy(path), tmp, callbacks) except Exception as e: - print_exception(e) - print_info('Path: %s' % str(current_path)) - print_info('Key = %s' % str(key) if 'key' in locals() else 'not defined') - print_info('Value = %s' % str(value) if 'value' in locals() else 'not defined') - print_info('Path = %s' % path) + print_exception(e, {'current path': '{}'.format(current_path), + 'key': '{}'.format(key if 'key' in locals() else 'not defined'), + 'value': '{}'.format(value if 'value' in locals() else 'not defined'), + 'path': '{}'.format(path), + } + ) From e7e6b92b59aa15c7167fe2e87dda5ae2303991ea Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Mon, 25 Mar 2019 10:48:53 +0100 Subject: [PATCH 563/667] Prettier output --- ScoutSuite/__main__.py | 9 ++++++--- ScoutSuite/providers/base/configs/services.py | 3 ++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/ScoutSuite/__main__.py b/ScoutSuite/__main__.py index 8eb8eec56..583f82fba 100644 --- a/ScoutSuite/__main__.py +++ b/ScoutSuite/__main__.py @@ -22,6 +22,7 @@ async def main(args=None): :return: """ + if not args: parser = ScoutSuiteArgumentParser() args = parser.parse_args() @@ -32,8 +33,10 @@ async def main(args=None): # Configure the debug level set_config_debug_level(args.get('debug')) + print_info('Launching Scout') + # Create a cloud provider object - print_info('Creating {} connector'.format(args.get('provider'))) + print_info('Creating provider instance') cloud_provider = get_provider(provider=args.get('provider'), profile=args.get('profile'), project_id=args.get('project_id'), @@ -57,7 +60,7 @@ async def main(args=None): # Complete run, including pulling data from provider if not args.get('fetch_local'): - print_info('Authenticating to {}'.format(args.get('provider'))) + print_info('Authenticating to {}'.format(cloud_provider.provider_name)) # Authenticate to the cloud provider authenticated = cloud_provider.authenticate(profile=args.get('profile'), user_account=args.get('user_account'), @@ -79,7 +82,7 @@ async def main(args=None): # Fetch data from provider APIs try: - print_info('Fetching data from provider APIs') + print_info('Gathering data from APIs') await cloud_provider.fetch(regions=args.get('regions')) except KeyboardInterrupt: print_info('\nCancelled by user') diff --git a/ScoutSuite/providers/base/configs/services.py b/ScoutSuite/providers/base/configs/services.py index 460c4a186..8d6fd1262 100644 --- a/ScoutSuite/providers/base/configs/services.py +++ b/ScoutSuite/providers/base/configs/services.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +from ScoutSuite.utils import format_service_name from ScoutSuite.core.console import print_error, print_exception, print_debug, print_info from ScoutSuite.providers.aws.utils import get_partition_name @@ -20,7 +21,7 @@ async def fetch(self, credentials, services=None, regions=None): # skip services if services != [] and service not in services: continue - print_info('Fetching resources for the {} service'.format(service)) # TODO pretty name + print_info('Fetching resources for the {} service'.format(format_service_name(service))) service_config = getattr(self, service) # call fetch method for the service if 'fetch_all' in dir(service_config): From 3ba82ea5992132b7a72575376c3e50be56b83313 Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Mon, 25 Mar 2019 11:10:20 +0100 Subject: [PATCH 564/667] Remove useless output --- ScoutSuite/__main__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ScoutSuite/__main__.py b/ScoutSuite/__main__.py index 583f82fba..1bfb6e12d 100644 --- a/ScoutSuite/__main__.py +++ b/ScoutSuite/__main__.py @@ -36,7 +36,6 @@ async def main(args=None): print_info('Launching Scout') # Create a cloud provider object - print_info('Creating provider instance') cloud_provider = get_provider(provider=args.get('provider'), profile=args.get('profile'), project_id=args.get('project_id'), @@ -54,7 +53,6 @@ async def main(args=None): # TODO: move this to after authentication, so that the report can be more specific to what's being scanned. # For example if scanning with a GCP service account, the SA email can only be known after authenticating... # Create a new report - print_info('Initializing report') report = Scout2Report(args.get('provider'), report_file_name, args.get( 'report_dir'), args.get('timestamp')) From d8dea8050613fea092b6d1ce938a29a0697cb568 Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Mon, 25 Mar 2019 11:49:07 +0100 Subject: [PATCH 565/667] Improved implementation and added writing exceptions to disk --- ScoutSuite/__init__.py | 2 ++ ScoutSuite/__main__.py | 2 +- ScoutSuite/core/console.py | 26 ++++++++++++------- ScoutSuite/output/html.py | 7 +++-- ScoutSuite/output/utils.py | 5 +++- ScoutSuite/providers/base/configs/services.py | 4 +-- 6 files changed, 31 insertions(+), 15 deletions(-) diff --git a/ScoutSuite/__init__.py b/ScoutSuite/__init__.py index 35dd7f96a..4a62b885d 100644 --- a/ScoutSuite/__init__.py +++ b/ScoutSuite/__init__.py @@ -7,6 +7,7 @@ EXCEPTIONS = 4242 HTMLREPORT = 424242 AWSRULESET = 42424242 +ERRORSLOG = 4242424242 DEFAULT_REPORT_DIR = 'scoutsuite-report' AWSCONFIG_FILE = 'scoutsuite-results/scoutsuite_results.js' @@ -14,5 +15,6 @@ EXCEPTIONS_FILE = 'scoutsuite-results/scoutsuite_exceptions.js' HTMLREPORT_FILE = 'report.html' GENERATOR_FILE = 'ruleset-generator.html' +ERRORSLOG_FILE = 'scoutsuite-results/scoutsuite_errors_log.json' REPORT_TITLE = 'Scout Suite Report' diff --git a/ScoutSuite/__main__.py b/ScoutSuite/__main__.py index 1bfb6e12d..628c926e7 100644 --- a/ScoutSuite/__main__.py +++ b/ScoutSuite/__main__.py @@ -126,7 +126,7 @@ async def main(args=None): processing_engine = ProcessingEngine(filter_rules) processing_engine.run(cloud_provider) - # Handle exceptions + print_info('Applying exceptions') try: exceptions = RuleExceptions(args.get('profile'), args.get('exceptions')[0]) exceptions.process(cloud_provider) diff --git a/ScoutSuite/core/console.py b/ScoutSuite/core/console.py index 429c1dc4d..816a2b3f4 100644 --- a/ScoutSuite/core/console.py +++ b/ScoutSuite/core/console.py @@ -14,7 +14,6 @@ logger = logging.getLogger('scout') # TODO logger should name the file where the error occurred -errors_list = [] exceptions_list = [] def set_config_debug_level(is_debug): @@ -42,18 +41,27 @@ def print_error(msg): logger.error(msg) -def print_exception(exception, additional_details): - logger.error(exception) +def print_exception(exception, additional_details=None): + + exc_type, exc_obj, exc_tb = sys.exc_info() + fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] + + str = '{} L{}: {}'.format(fname, exc_tb.tb_lineno, exception) + global verbose_exceptions + if verbose_exceptions: + logger.exception(str) + else: + logger.error(str) + + # logger.error('{} L{}: {}'.format(fname, exc_tb.tb_lineno, exception)) - # TODO do something with these global exceptions_list - exceptions_list.append({'exception': exception, - 'traceback': traceback.format_exc(), + exceptions_list.append({'file': fname, + 'line': exc_tb.tb_lineno, + 'exception': '{}'.format(exception), + 'traceback': '{}'.format(traceback.format_exc()), 'additional_details': additional_details}) - a = exceptions_list - b = 1 - def print_info(msg): diff --git a/ScoutSuite/output/html.py b/ScoutSuite/output/html.py index 381d1eff7..0d548f8b0 100644 --- a/ScoutSuite/output/html.py +++ b/ScoutSuite/output/html.py @@ -9,10 +9,11 @@ import dateutil.tz from ScoutSuite.core.console import print_info, print_exception -from ScoutSuite import AWSCONFIG, EXCEPTIONS, HTMLREPORT, AWSRULESET, AWSCONFIG_FILE, EXCEPTIONS_FILE, HTMLREPORT_FILE, \ - GENERATOR_FILE +from ScoutSuite import AWSCONFIG, EXCEPTIONS, HTMLREPORT, AWSRULESET, ERRORSLOG, AWSCONFIG_FILE, EXCEPTIONS_FILE, HTMLREPORT_FILE, \ + GENERATOR_FILE, ERRORSLOG_FILE from ScoutSuite.output.js import JavaScriptReaderWriter from ScoutSuite.output.utils import get_filename, prompt_4_overwrite +from ScoutSuite.core.console import exceptions_list class HTMLReport(object): @@ -80,6 +81,8 @@ def save(self, config, exceptions, force_write=False, debug=False): self.prepare_html_report_dir() self.jsrw.save_to_file(config, AWSCONFIG, force_write, debug) self.jsrw.save_to_file(exceptions, EXCEPTIONS, force_write, debug) + global exceptions_list + self.jsrw.save_to_file(exceptions_list, ERRORSLOG, force_write, debug=True) return self.create_html_report(force_write) def create_html_report(self, force_write): diff --git a/ScoutSuite/output/utils.py b/ScoutSuite/output/utils.py index 322c51d12..3d97e31cc 100644 --- a/ScoutSuite/output/utils.py +++ b/ScoutSuite/output/utils.py @@ -8,7 +8,7 @@ from six.moves import input -from ScoutSuite import AWSCONFIG, EXCEPTIONS, HTMLREPORT, AWSRULESET, AWSCONFIG_FILE, EXCEPTIONS_FILE, HTMLREPORT_FILE, AWSRULESET_FILE +from ScoutSuite import AWSCONFIG, EXCEPTIONS, HTMLREPORT, AWSRULESET, ERRORSLOG, AWSCONFIG_FILE, EXCEPTIONS_FILE, HTMLREPORT_FILE, AWSRULESET_FILE, ERRORSLOG_FILE def prompt_4_yes_no(question): @@ -57,6 +57,9 @@ def get_filename(config_type, profile, report_dir): elif config_type == AWSRULESET: filename = AWSRULESET_FILE first_line = 'scoutsuite_results =' + elif config_type == ERRORSLOG: + filename = ERRORSLOG_FILE + first_line = None else: print_error('invalid config type provided (%s)' % config_type) raise Exception diff --git a/ScoutSuite/providers/base/configs/services.py b/ScoutSuite/providers/base/configs/services.py index 8d6fd1262..59aa11da1 100644 --- a/ScoutSuite/providers/base/configs/services.py +++ b/ScoutSuite/providers/base/configs/services.py @@ -35,7 +35,7 @@ async def fetch(self, credentials, services=None, regions=None): if hasattr(service_config, 'finalize'): await service_config.finalize() else: - print_debug('No method to fetch service %s.' % service) + print_debug('No method to fetch the {} service'.format(format_service_name(service))) except Exception as e: - print_error('Error: could not fetch %s configuration.' % service) + print_error('Could not fetch the {} service'.format(format_service_name(service))) print_exception(e) From 4b7f7a77917882f5a57303c059e72765efc68727 Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Mon, 25 Mar 2019 13:21:30 +0100 Subject: [PATCH 566/667] Only load exceptions included in arguments --- ScoutSuite/__main__.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/ScoutSuite/__main__.py b/ScoutSuite/__main__.py index 628c926e7..f2f8449a1 100644 --- a/ScoutSuite/__main__.py +++ b/ScoutSuite/__main__.py @@ -127,14 +127,16 @@ async def main(args=None): processing_engine.run(cloud_provider) print_info('Applying exceptions') - try: - exceptions = RuleExceptions(args.get('profile'), args.get('exceptions')[0]) - exceptions.process(cloud_provider) - exceptions = exceptions.exceptions - except Exception as e: - print_debug( - 'Warning, failed to load exceptions. The file may not exist or may have an invalid format.') - exceptions = {} + if args.get('exceptions')[0]: + try: + exceptions = RuleExceptions(args.get('profile'), args.get('exceptions')[0]) + exceptions.process(cloud_provider) + exceptions = exceptions.exceptions + except Exception as e: + print_debug('Failed to load exceptions. The file may not exist or may have an invalid format.') + exceptions = {} + else: + exceptions = {} # Finalize cloud_provider.postprocessing(report.current_time, finding_rules) From acd2ef708d18d81e6b29b6a14c23ccaacbc51493 Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Mon, 25 Mar 2019 13:23:36 +0100 Subject: [PATCH 567/667] Only write exceptions file if not empty --- ScoutSuite/output/html.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/output/html.py b/ScoutSuite/output/html.py index 0d548f8b0..4fbe7310c 100644 --- a/ScoutSuite/output/html.py +++ b/ScoutSuite/output/html.py @@ -82,7 +82,8 @@ def save(self, config, exceptions, force_write=False, debug=False): self.jsrw.save_to_file(config, AWSCONFIG, force_write, debug) self.jsrw.save_to_file(exceptions, EXCEPTIONS, force_write, debug) global exceptions_list - self.jsrw.save_to_file(exceptions_list, ERRORSLOG, force_write, debug=True) + if exceptions_list: + self.jsrw.save_to_file(exceptions_list, ERRORSLOG, force_write, debug=True) return self.create_html_report(force_write) def create_html_report(self, force_write): From 767bbe278880cad6d1147b8da36d3f56e461f0dc Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Mon, 25 Mar 2019 13:24:06 +0100 Subject: [PATCH 568/667] Move print --- ScoutSuite/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/__main__.py b/ScoutSuite/__main__.py index f2f8449a1..d9729091a 100644 --- a/ScoutSuite/__main__.py +++ b/ScoutSuite/__main__.py @@ -126,8 +126,8 @@ async def main(args=None): processing_engine = ProcessingEngine(filter_rules) processing_engine.run(cloud_provider) - print_info('Applying exceptions') if args.get('exceptions')[0]: + print_info('Applying exceptions') try: exceptions = RuleExceptions(args.get('profile'), args.get('exceptions')[0]) exceptions.process(cloud_provider) From 2d2a42c2ea787c8fb5cc4514b442a98a46596a60 Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Mon, 25 Mar 2019 13:28:27 +0100 Subject: [PATCH 569/667] Cleaning up output --- ScoutSuite/__main__.py | 2 +- ScoutSuite/output/html.py | 2 +- ScoutSuite/output/js.py | 6 ++---- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/ScoutSuite/__main__.py b/ScoutSuite/__main__.py index d9729091a..b8ba7c872 100644 --- a/ScoutSuite/__main__.py +++ b/ScoutSuite/__main__.py @@ -147,7 +147,7 @@ async def main(args=None): # Open the report by default if not args.get('no_browser'): - print_info('Opening the HTML report...') + print_info('Opening the HTML report') url = 'file://%s' % os.path.abspath(html_report_path) webbrowser.open(url, new=2) diff --git a/ScoutSuite/output/html.py b/ScoutSuite/output/html.py index 4fbe7310c..076d0f707 100644 --- a/ScoutSuite/output/html.py +++ b/ScoutSuite/output/html.py @@ -95,7 +95,7 @@ def create_html_report(self, force_write): contents += self.get_content_from('summaries') contents += self.get_content_from('summaries/%s' % self.provider) new_file, first_line = get_filename(HTMLREPORT, self.profile, self.report_dir) - print_info('Creating %s ...' % new_file) + print_info('Creating %s' % new_file) if prompt_4_overwrite(new_file, force_write): if os.path.exists(new_file): os.remove(new_file) diff --git a/ScoutSuite/output/js.py b/ScoutSuite/output/js.py index c56d40853..350645709 100644 --- a/ScoutSuite/output/js.py +++ b/ScoutSuite/output/js.py @@ -65,7 +65,7 @@ def save_to_file(self, config, config_type, force_write, debug): config_path, first_line = get_filename(config_type, self.profile, self.report_dir) print_info('Saving data to %s' % config_path) try: - with self.__open_file(config_path, force_write, False) as f: + with self.__open_file(config_path, force_write) as f: if first_line: print('%s' % first_line, file=f) print('%s' % json.dumps(config, indent=4 if debug else None, separators=(',', ': '), sort_keys=True, cls=Scout2Encoder), file=f) @@ -77,7 +77,7 @@ def to_dict(self, config): return json.loads(json.dumps(config, separators=(',', ': '), cls=Scout2Encoder)) - def __open_file(self, config_filename, force_write, quiet=False): + def __open_file(self, config_filename, force_write): """ :param config_filename: @@ -85,8 +85,6 @@ def __open_file(self, config_filename, force_write, quiet=False): :param quiet: :return: """ - if not quiet: - print_info('Saving config...') if prompt_4_overwrite(config_filename, force_write): try: config_dirname = os.path.dirname(config_filename) From 4e0426a294e19caef4d55ab924b1c24c24e1f53b Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 25 Mar 2019 09:49:20 -0400 Subject: [PATCH 570/667] Update ScoutSuite/providers/aws/facade/facade.py --- ScoutSuite/providers/aws/facade/facade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/facade/facade.py b/ScoutSuite/providers/aws/facade/facade.py index 9647e1e7c..58cd81497 100644 --- a/ScoutSuite/providers/aws/facade/facade.py +++ b/ScoutSuite/providers/aws/facade/facade.py @@ -19,7 +19,7 @@ try: from ScoutSuite.providers.aws.facade.config_private import ConfigFacade from ScoutSuite.providers.aws.facade.kms_private import KMSFacade -except NameError: +except ImportError: pass From 23afc642168ad5f8858c5340c43673213b08caae Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 25 Mar 2019 11:06:43 -0400 Subject: [PATCH 571/667] Fixed bad merge --- ScoutSuite/providers/aws/configs/services.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index f95cdaf49..1e39d5087 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -7,9 +7,8 @@ from ScoutSuite.providers.aws.resources.directconnect.service import DirectConnect from ScoutSuite.providers.aws.resources.ec2.service import EC2 from ScoutSuite.providers.aws.resources.efs.service import EFS -from ScoutSuite.providers.aws.resources.elb.service import ELB from ScoutSuite.providers.aws.resources.elasticache.service import ElastiCache -from ScoutSuite.providers.aws.services.elb import ELBConfig +from ScoutSuite.providers.aws.resources.elb.service import ELB from ScoutSuite.providers.aws.resources.elbv2.service import ELBv2 from ScoutSuite.providers.aws.resources.emr.service import EMR from ScoutSuite.providers.aws.services.iam import IAMConfig From fca45ddfedd7706224ee9ce67773aafd1dd68a78 Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 25 Mar 2019 17:55:46 -0400 Subject: [PATCH 572/667] Fix imports. --- tests/test-rules-processingengine.py | 8 +++----- tests/test-rules-ruleset.py | 7 ++----- tests/test-scoutsuite.py | 9 +++------ 3 files changed, 8 insertions(+), 16 deletions(-) diff --git a/tests/test-rules-processingengine.py b/tests/test-rules-processingengine.py index e6cff5799..d76e9e7b4 100644 --- a/tests/test-rules-processingengine.py +++ b/tests/test-rules-processingengine.py @@ -1,11 +1,8 @@ -# -*- coding: utf-8 -*- - import json import os import tempfile -from ScoutSuite.core.console import config_debug_level, print_error - +from ScoutSuite.core.console import set_config_debug_level, print_error from ScoutSuite.core.processingengine import ProcessingEngine from ScoutSuite.core.ruleset import Ruleset @@ -17,7 +14,7 @@ class DummyObject(object): class TestAWSScout2RulesProcessingEngine: def setup(self): - config_debug_level(True) + set_config_debug_level(True) self.rule_counters = {'found': 0, 'tested': 0, 'verified': 0} self.test_dir = os.path.dirname(os.path.realpath(__file__)) @@ -35,6 +32,7 @@ def test_all_finding_rules(self): self.rule_counters['found'] += 1 rule = ruleset['rules'][rule_file_name][0] rule['enabled'] = True + print(rule_file_name) self._test_rule(ruleset_file_name, rule_file_name, rule) print('Existing rules: %d' % self.rule_counters['found']) diff --git a/tests/test-rules-ruleset.py b/tests/test-rules-ruleset.py index 7f3c4be16..d245344c6 100644 --- a/tests/test-rules-ruleset.py +++ b/tests/test-rules-ruleset.py @@ -1,10 +1,7 @@ -# -*- coding: utf-8 -*- - import os from mock import patch -from ScoutSuite.core.console import config_debug_level, print_debug - +from ScoutSuite.core.console import set_config_debug_level, print_debug from ScoutSuite.core.rule import Rule from ScoutSuite.core.ruleset import Ruleset @@ -12,7 +9,7 @@ class TestAWSScout2RulesRuleset: def setup(self): - config_debug_level(True) + set_config_debug_level(True) self.test_dir = os.path.dirname(os.path.realpath(__file__)) self.test_ruleset_001 = os.path.join(self.test_dir, 'data/test-ruleset.json') diff --git a/tests/test-scoutsuite.py b/tests/test-scoutsuite.py index 4cc5c824a..3435b37c9 100644 --- a/tests/test-scoutsuite.py +++ b/tests/test-scoutsuite.py @@ -1,19 +1,16 @@ import subprocess - import mock -from nose.plugins.attrib import attr +from nose.plugins.attrib import attr from ScoutSuite.__main__ import * +from ScoutSuite.core.console import set_config_debug_level -# -# Test for Scout.py -# class TestScoutSuiteClass: @classmethod def setUpClass(cls): - config_debug_level(True) + set_config_debug_level(True) cls.has_run_scout_suite = False @staticmethod From f56f76b215a67ac110df000bd63b0817a7b3726a Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 25 Mar 2019 23:58:13 -0400 Subject: [PATCH 573/667] Fix get_client calls. --- ScoutSuite/providers/aws/facade/ec2.py | 4 ++-- ScoutSuite/providers/aws/facade/elb.py | 2 +- ScoutSuite/providers/aws/facade/elbv2.py | 2 +- ScoutSuite/providers/aws/facade/rds.py | 4 ++-- ScoutSuite/providers/aws/facade/ses.py | 4 ++-- ScoutSuite/providers/aws/facade/sns.py | 2 +- ScoutSuite/providers/aws/facade/sqs.py | 4 ++-- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/ec2.py b/ScoutSuite/providers/aws/facade/ec2.py index e15fed6a9..288cefeec 100644 --- a/ScoutSuite/providers/aws/facade/ec2.py +++ b/ScoutSuite/providers/aws/facade/ec2.py @@ -40,7 +40,7 @@ async def get_security_groups(self, region: str, vpc: str): 'ec2', region, self.session, 'describe_security_groups', 'SecurityGroups', Filters=filters) async def get_vpcs(self, region: str): - ec2_client = AWSFacadeUtils.get_client('ec2', region, self.session) + ec2_client = AWSFacadeUtils.get_client('ec2', self.session, region) return await run_concurrently(lambda: ec2_client.describe_vpcs()['Vpcs']) async def get_images(self, region: str, owner_id: str): @@ -89,7 +89,7 @@ async def cache_flow_logs(self, region: str): await AWSFacadeUtils.get_all_pages('ec2', region, self.session, 'describe_flow_logs', 'FlowLogs') async def get_subnets(self, region: str, vpc: str): - ec2_client = AWSFacadeUtils.get_client('ec2', region, self.session) + ec2_client = AWSFacadeUtils.get_client('ec2', self.session, region) filters = [{'Name': 'vpc-id', 'Values': [vpc]}] subnets = await run_concurrently( lambda: ec2_client.describe_subnets(Filters=filters)['Subnets'] diff --git a/ScoutSuite/providers/aws/facade/elb.py b/ScoutSuite/providers/aws/facade/elb.py index 3ad1072b7..71eb7b4e9 100644 --- a/ScoutSuite/providers/aws/facade/elb.py +++ b/ScoutSuite/providers/aws/facade/elb.py @@ -37,7 +37,7 @@ async def cache_load_balancers(self, region): await asyncio.wait(tasks) async def _get_and_set_load_balancer_attributes(self, region: str, load_balancer: {}): - elb_client = AWSFacadeUtils.get_client('elb', region, self.session) + elb_client = AWSFacadeUtils.get_client('elb', self.session, region) load_balancer['attributes'] = await run_concurrently( lambda: elb_client.describe_load_balancer_attributes( LoadBalancerName=load_balancer['LoadBalancerName'])['LoadBalancerAttributes'] diff --git a/ScoutSuite/providers/aws/facade/elbv2.py b/ScoutSuite/providers/aws/facade/elbv2.py index e407089f8..b46603d5d 100644 --- a/ScoutSuite/providers/aws/facade/elbv2.py +++ b/ScoutSuite/providers/aws/facade/elbv2.py @@ -28,7 +28,7 @@ async def cache_load_balancers(self, region): load_balancer['VpcId'] if 'VpcId' in load_balancer and load_balancer['VpcId'] else ec2_classic async def get_load_balancer_attributes(self, region: str, load_balancer_arn: str): - elbv2_client = AWSFacadeUtils.get_client('elbv2', region, self.session) + elbv2_client = AWSFacadeUtils.get_client('elbv2', self.session, region) return await run_concurrently( lambda: elbv2_client.describe_load_balancer_attributes( LoadBalancerArn=load_balancer_arn)['Attributes'] diff --git a/ScoutSuite/providers/aws/facade/rds.py b/ScoutSuite/providers/aws/facade/rds.py index 7a710ff5d..7c93fd7d8 100644 --- a/ScoutSuite/providers/aws/facade/rds.py +++ b/ScoutSuite/providers/aws/facade/rds.py @@ -26,7 +26,7 @@ async def _cache_instances(self, region: str): self._instances_cache[region] = await AWSFacadeUtils.get_all_pages('rds', region, self.session, 'describe_db_instances', 'DBInstances') - client = AWSFacadeUtils.get_client('rds', region, self.session) + client = AWSFacadeUtils.get_client('rds', self.session, region) for instance in self._instances_cache[region]: instance['VpcId'] = instance['DBSubnetGroup']['VpcId'] \ if 'DBSubnetGroup' in instance and 'VpcId' in instance['DBSubnetGroup'] \ @@ -48,7 +48,7 @@ async def _cache_snapshots(self, region: str): if region in self._snapshots_cache: return - client = AWSFacadeUtils.get_client('rds', region, self.session) + client = AWSFacadeUtils.get_client('rds', self.session, region) self._snapshots_cache[region] = await AWSFacadeUtils.get_all_pages('rds', region, self.session, 'describe_db_snapshots', 'DBSnapshots') for snapshot in self._snapshots_cache[region]: snapshot_id = snapshot['DBSnapshotIdentifier'] diff --git a/ScoutSuite/providers/aws/facade/ses.py b/ScoutSuite/providers/aws/facade/ses.py index e4762ad4e..8accf84f7 100644 --- a/ScoutSuite/providers/aws/facade/ses.py +++ b/ScoutSuite/providers/aws/facade/ses.py @@ -22,14 +22,14 @@ async def get_identities(self, region: str): return identities async def get_identity_dkim_attributes(self, region: str, identity_name: str): - ses_client = AWSFacadeUtils.get_client('ses', region, self.session) + ses_client = AWSFacadeUtils.get_client('ses', self.session, region) dkim_attributes = await run_concurrently( lambda: ses_client.get_identity_dkim_attributes(Identities=[identity_name])['DkimAttributes'][identity_name] ) return identity_name, dkim_attributes async def get_identity_policies(self, region: str, identity_name: str): - ses_client = AWSFacadeUtils.get_client('ses', region, self.session) + ses_client = AWSFacadeUtils.get_client('ses', self.session, region) policy_names = await run_concurrently( lambda: ses_client.list_identity_policies(Identity=identity_name)['PolicyNames'] ) diff --git a/ScoutSuite/providers/aws/facade/sns.py b/ScoutSuite/providers/aws/facade/sns.py index 96fb58869..24b749aee 100644 --- a/ScoutSuite/providers/aws/facade/sns.py +++ b/ScoutSuite/providers/aws/facade/sns.py @@ -26,7 +26,7 @@ async def get_topics(self, region: str): return topics async def get_and_set_topic_attributes(self, region: str, topic: {}): - sns_client = AWSFacadeUtils.get_client('sns', region, self.session) + sns_client = AWSFacadeUtils.get_client('sns', self.session, region) topic['attributes'] = await run_concurrently( lambda: sns_client.get_topic_attributes(TopicArn=topic['TopicArn'])['Attributes'] ) diff --git a/ScoutSuite/providers/aws/facade/sqs.py b/ScoutSuite/providers/aws/facade/sqs.py index b50accd76..80ef63a21 100644 --- a/ScoutSuite/providers/aws/facade/sqs.py +++ b/ScoutSuite/providers/aws/facade/sqs.py @@ -7,7 +7,7 @@ class SQSFacade(AWSBaseFacade): async def get_queues(self, region: str, attribute_names: []): - sqs_client = AWSFacadeUtils.get_client('sqs', region, self.session) + sqs_client = AWSFacadeUtils.get_client('sqs', self.session, region) raw_queues = await run_concurrently(sqs_client.list_queues) if 'QueueUrls' not in raw_queues: @@ -28,7 +28,7 @@ async def get_queues(self, region: str, attribute_names: []): return queues async def get_queue_attributes(self, region: str, queue_url: str, attribute_names: []): - sqs_client = AWSFacadeUtils.get_client('sqs', region, self.session) + sqs_client = AWSFacadeUtils.get_client('sqs', self.session, region) queue_attributes = await run_concurrently( lambda: sqs_client.get_queue_attributes(QueueUrl=queue_url, AttributeNames=attribute_names)['Attributes'] ) From e512f8631d32afe9afdf2e67101835f2c0711acf Mon Sep 17 00:00:00 2001 From: misg Date: Mon, 25 Mar 2019 23:59:20 -0400 Subject: [PATCH 574/667] Remove duplicated pre-processing calls. --- ScoutSuite/providers/aws/provider.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index 5ec345566..b43cdd3b6 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -79,8 +79,6 @@ def preprocessing(self, ip_ranges=None, ip_ranges_name_key=None): self._match_iam_policies_and_buckets() self._add_cidr_display_name(ip_ranges, ip_ranges_name_key) - self._merge_route53_and_route53domains() - self._match_iam_policies_and_buckets() super(AWSProvider, self).preprocessing() From 410f07040c587f24281c0835201f4a87130bdafb Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Tue, 26 Mar 2019 09:22:34 +0100 Subject: [PATCH 575/667] Remove comment --- ScoutSuite/core/console.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ScoutSuite/core/console.py b/ScoutSuite/core/console.py index 816a2b3f4..9ff954c16 100644 --- a/ScoutSuite/core/console.py +++ b/ScoutSuite/core/console.py @@ -53,8 +53,6 @@ def print_exception(exception, additional_details=None): else: logger.error(str) - # logger.error('{} L{}: {}'.format(fname, exc_tb.tb_lineno, exception)) - global exceptions_list exceptions_list.append({'file': fname, 'line': exc_tb.tb_lineno, From fd48a5841dd8005ae19c1bbf2fe27dcec2453d79 Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Tue, 26 Mar 2019 09:53:24 +0100 Subject: [PATCH 576/667] Get rid of error codes --- ScoutSuite/__init__.py | 22 +++++------- ScoutSuite/__main__.py | 10 +++--- ScoutSuite/core/console.py | 16 ++++----- ScoutSuite/core/exceptions.py | 4 +-- ScoutSuite/output/html.py | 34 +++++++++--------- ScoutSuite/output/js.py | 8 ++--- ScoutSuite/output/utils.py | 65 +++++++++++++++++++++-------------- 7 files changed, 82 insertions(+), 77 deletions(-) diff --git a/ScoutSuite/__init__.py b/ScoutSuite/__init__.py index 4a62b885d..66e4dd7f4 100644 --- a/ScoutSuite/__init__.py +++ b/ScoutSuite/__init__.py @@ -3,18 +3,14 @@ __author__ = 'NCC Group' __version__ = '4.3.0' -AWSCONFIG = 42 -EXCEPTIONS = 4242 -HTMLREPORT = 424242 -AWSRULESET = 42424242 -ERRORSLOG = 4242424242 - DEFAULT_REPORT_DIR = 'scoutsuite-report' -AWSCONFIG_FILE = 'scoutsuite-results/scoutsuite_results.js' -AWSRULESET_FILE = 'scoutsuite-results/scoutsuite_ruleset.js' -EXCEPTIONS_FILE = 'scoutsuite-results/scoutsuite_exceptions.js' -HTMLREPORT_FILE = 'report.html' -GENERATOR_FILE = 'ruleset-generator.html' -ERRORSLOG_FILE = 'scoutsuite-results/scoutsuite_errors_log.json' +DEFAULT_RESULT_FILE = 'scoutsuite-results/scoutsuite_results.js' +DEFAULT_RULESET_FILE = 'scoutsuite-results/scoutsuite_ruleset.js' +DEFAULT_EXCEPTIONS_FILE = 'scoutsuite-results/scoutsuite_exceptions.js' +DEFAULT_ERRORS_FILE = 'scoutsuite-results/scoutsuite_errors_log.json' +DEFAULT_HTMLREPORT_FILE = 'report.html' +DEFAULT_GENERATOR_FILE = 'ruleset-generator.html' + +ERRORS_LIST = [] -REPORT_TITLE = 'Scout Suite Report' +DEFAULT_REPORT_TITLE = 'Scout Suite Report' \ No newline at end of file diff --git a/ScoutSuite/__main__.py b/ScoutSuite/__main__.py index b8ba7c872..ad0b8ab4a 100644 --- a/ScoutSuite/__main__.py +++ b/ScoutSuite/__main__.py @@ -5,7 +5,7 @@ import os import webbrowser -from ScoutSuite import AWSCONFIG +from ScoutSuite import DEFAULT_RESULT_FILE from ScoutSuite.core.cli_parser import ScoutSuiteArgumentParser from ScoutSuite.core.console import set_config_debug_level, print_info, print_debug, print_error from ScoutSuite.core.exceptions import RuleExceptions @@ -19,8 +19,6 @@ async def main(args=None): """ Main method that runs a scan - - :return: """ if not args: @@ -90,7 +88,7 @@ async def main(args=None): if args.get('update'): print_info('Updating existing data') current_run_services = copy.deepcopy(cloud_provider.services) - last_run_dict = report.jsrw.load_from_file(AWSCONFIG) + last_run_dict = report.jsrw.load_from_file(DEFAULT_RESULT_FILE) cloud_provider.services = last_run_dict['services'] for service in cloud_provider.service_list: cloud_provider.services[service] = current_run_services[service] @@ -99,7 +97,7 @@ async def main(args=None): else: print_info('Using local data') # Reload to flatten everything into a python dictionary - last_run_dict = report.jsrw.load_from_file(AWSCONFIG) + last_run_dict = report.jsrw.load_from_file(DEFAULT_RESULT_FILE) for key in last_run_dict: setattr(cloud_provider, key, last_run_dict[key]) @@ -155,7 +153,7 @@ async def main(args=None): def generate_report_name(provider_code, args): - # TODO this should be done within the provider + # TODO this should be done within the providers # A pre-requisite to this is to generate report AFTER authentication if provider_code == 'aws': if args.get('profile'): diff --git a/ScoutSuite/core/console.py b/ScoutSuite/core/console.py index 9ff954c16..daf7c51c7 100644 --- a/ScoutSuite/core/console.py +++ b/ScoutSuite/core/console.py @@ -6,15 +6,14 @@ import coloredlogs, logging +from ScoutSuite import ERRORS_LIST + ######################################## # Output configuration ######################################## verbose_exceptions = False logger = logging.getLogger('scout') -# TODO logger should name the file where the error occurred - -exceptions_list = [] def set_config_debug_level(is_debug): """ @@ -53,12 +52,11 @@ def print_exception(exception, additional_details=None): else: logger.error(str) - global exceptions_list - exceptions_list.append({'file': fname, - 'line': exc_tb.tb_lineno, - 'exception': '{}'.format(exception), - 'traceback': '{}'.format(traceback.format_exc()), - 'additional_details': additional_details}) + ERRORS_LIST.append({'file': fname, + 'line': exc_tb.tb_lineno, + 'exception': '{}'.format(exception), + 'traceback': '{}'.format(traceback.format_exc()), + 'additional_details': additional_details}) def print_info(msg): diff --git a/ScoutSuite/core/exceptions.py b/ScoutSuite/core/exceptions.py index d2b477068..490c99b94 100644 --- a/ScoutSuite/core/exceptions.py +++ b/ScoutSuite/core/exceptions.py @@ -5,7 +5,7 @@ from ScoutSuite.core.console import print_debug -from ScoutSuite import EXCEPTIONS +from ScoutSuite import DEFAULT_EXCEPTIONS_FILE from ScoutSuite.output.js import JavaScriptReaderWriter @@ -15,7 +15,7 @@ def __init__(self, profile, file_path=None): self.profile = profile self.file_path = file_path self.jsrw = JavaScriptReaderWriter(self.profile) - self.exceptions = self.jsrw.load_from_file(config_type=EXCEPTIONS, + self.exceptions = self.jsrw.load_from_file(file_type=DEFAULT_EXCEPTIONS_FILE, config_path=self.file_path, first_line=True) diff --git a/ScoutSuite/output/html.py b/ScoutSuite/output/html.py index 076d0f707..e60b27d23 100644 --- a/ScoutSuite/output/html.py +++ b/ScoutSuite/output/html.py @@ -9,11 +9,10 @@ import dateutil.tz from ScoutSuite.core.console import print_info, print_exception -from ScoutSuite import AWSCONFIG, EXCEPTIONS, HTMLREPORT, AWSRULESET, ERRORSLOG, AWSCONFIG_FILE, EXCEPTIONS_FILE, HTMLREPORT_FILE, \ - GENERATOR_FILE, ERRORSLOG_FILE +from ScoutSuite import DEFAULT_RESULT_FILE, DEFAULT_HTMLREPORT_FILE, DEFAULT_GENERATOR_FILE, DEFAULT_EXCEPTIONS_FILE +from ScoutSuite import ERRORS_LIST from ScoutSuite.output.js import JavaScriptReaderWriter from ScoutSuite.output.utils import get_filename, prompt_4_overwrite -from ScoutSuite.core.console import exceptions_list class HTMLReport(object): @@ -73,17 +72,16 @@ class Scout2Report(HTMLReport): def __init__(self, provider, profile=None, report_dir=None, timestamp=False, exceptions=None): exceptions = {} if exceptions is None else exceptions - self.html_root = HTMLREPORT_FILE + self.html_root = DEFAULT_HTMLREPORT_FILE self.provider = provider super(Scout2Report, self).__init__(profile, report_dir, timestamp, exceptions) def save(self, config, exceptions, force_write=False, debug=False): self.prepare_html_report_dir() - self.jsrw.save_to_file(config, AWSCONFIG, force_write, debug) - self.jsrw.save_to_file(exceptions, EXCEPTIONS, force_write, debug) - global exceptions_list - if exceptions_list: - self.jsrw.save_to_file(exceptions_list, ERRORSLOG, force_write, debug=True) + self.jsrw.save_to_file(config, 'RESULTS', force_write, debug) + self.jsrw.save_to_file(exceptions, 'EXCEPTIONS', force_write, debug) + if ERRORS_LIST: + self.jsrw.save_to_file(ERRORS_LIST, 'ERRORS', force_write, debug=True) return self.create_html_report(force_write) def create_html_report(self, force_write): @@ -94,7 +92,7 @@ def create_html_report(self, force_write): # Use all scripts under html/summaries/ contents += self.get_content_from('summaries') contents += self.get_content_from('summaries/%s' % self.provider) - new_file, first_line = get_filename(HTMLREPORT, self.profile, self.report_dir) + new_file, first_line = get_filename('HTMLREPORT', self.profile, self.report_dir) print_info('Creating %s' % new_file) if prompt_4_overwrite(new_file, force_write): if os.path.exists(new_file): @@ -104,10 +102,10 @@ def create_html_report(self, force_write): for line in f: newline = line if self.profile != 'default': - newline = newline.replace(AWSCONFIG_FILE, - AWSCONFIG_FILE.replace('.js', '-%s.js' % self.profile)) - newline = newline.replace(EXCEPTIONS_FILE, - EXCEPTIONS_FILE.replace('.js', '-%s.js' % self.profile)) + newline = newline.replace(DEFAULT_RESULT_FILE, + DEFAULT_RESULT_FILE.replace('.js', '-%s.js' % self.profile)) + newline = newline.replace(DEFAULT_EXCEPTIONS_FILE, + DEFAULT_EXCEPTIONS_FILE.replace('.js', '-%s.js' % self.profile)) newline = newline.replace('', contents) nf.write(newline) return new_file @@ -120,17 +118,17 @@ class RulesetGenerator(HTMLReport): def __init__(self, ruleset_name, report_dir=None, timestamp=False, exceptions=None): exceptions = {} if exceptions is None else exceptions - self.html_root = GENERATOR_FILE + self.html_root = DEFAULT_GENERATOR_FILE self.ruleset_name = ruleset_name super(RulesetGenerator, self).__init__(ruleset_name, report_dir, timestamp, exceptions) def create_html_report(self, force_write): - src_rule_generator = os.path.join(self.html_data_path, GENERATOR_FILE) - rule_generator = os.path.join(self.report_dir, GENERATOR_FILE) + src_rule_generator = os.path.join(self.html_data_path, DEFAULT_GENERATOR_FILE) + rule_generator = os.path.join(self.report_dir, DEFAULT_GENERATOR_FILE) shutil.copyfile(src_rule_generator, rule_generator) return rule_generator def save(self, config, force_write=False, debug=False): self.prepare_html_report_dir() - self.jsrw.save_to_file(config, AWSRULESET, force_write, debug) + self.jsrw.save_to_file(config, 'RULESET', force_write, debug) return self.create_html_report(force_write) diff --git a/ScoutSuite/output/js.py b/ScoutSuite/output/js.py index 350645709..093a98bff 100644 --- a/ScoutSuite/output/js.py +++ b/ScoutSuite/output/js.py @@ -50,9 +50,9 @@ def __init__(self, profile, report_dir=None, timestamp=None): self.timestamp = self.current_time.strftime("%Y-%m-%d_%Hh%M%z") if not timestamp else timestamp - def load_from_file(self, config_type, config_path = None, first_line = None): + def load_from_file(self, file_type, config_path = None, first_line = None): if not config_path: - config_path, first_line = get_filename(config_type, self.profile, self.report_dir) + config_path, first_line = get_filename(file_type, self.profile, self.report_dir) with open(config_path, 'rt') as f: json_payload = f.readlines() if first_line: @@ -61,8 +61,8 @@ def load_from_file(self, config_type, config_path = None, first_line = None): return json.loads(json_payload) - def save_to_file(self, config, config_type, force_write, debug): - config_path, first_line = get_filename(config_type, self.profile, self.report_dir) + def save_to_file(self, config, file_type, force_write, debug): + config_path, first_line = get_filename(file_type, self.profile, self.report_dir) print_info('Saving data to %s' % config_path) try: with self.__open_file(config_path, force_write) as f: diff --git a/ScoutSuite/output/utils.py b/ScoutSuite/output/utils.py index 3d97e31cc..fd38cf65f 100644 --- a/ScoutSuite/output/utils.py +++ b/ScoutSuite/output/utils.py @@ -8,7 +8,8 @@ from six.moves import input -from ScoutSuite import AWSCONFIG, EXCEPTIONS, HTMLREPORT, AWSRULESET, ERRORSLOG, AWSCONFIG_FILE, EXCEPTIONS_FILE, HTMLREPORT_FILE, AWSRULESET_FILE, ERRORSLOG_FILE +from ScoutSuite import DEFAULT_RESULT_FILE, DEFAULT_HTMLREPORT_FILE, DEFAULT_EXCEPTIONS_FILE, DEFAULT_RULESET_FILE, \ + DEFAULT_ERRORS_FILE def prompt_4_yes_no(question): @@ -44,27 +45,41 @@ def prompt_4_overwrite(filename, force_write): return prompt_4_yes_no('File \'{}\' already exists. Do you want to overwrite it'.format(filename)) -def get_filename(config_type, profile, report_dir): - if config_type == AWSCONFIG: - filename = AWSCONFIG_FILE - first_line = 'scoutsuite_results =' - elif config_type == EXCEPTIONS: - filename = EXCEPTIONS_FILE - first_line = 'exceptions =' - elif config_type == HTMLREPORT: - filename = HTMLREPORT_FILE - first_line = None - elif config_type == AWSRULESET: - filename = AWSRULESET_FILE - first_line = 'scoutsuite_results =' - elif config_type == ERRORSLOG: - filename = ERRORSLOG_FILE - first_line = None - else: - print_error('invalid config type provided (%s)' % config_type) - raise Exception - # Append profile name if necessary - if profile != 'default' and config_type != AWSRULESET: - name, extention = filename.split('.') - filename = '%s-%s.%s' % (name, profile, extention) - return (os.path.join(report_dir, filename), first_line) +def get_filename(file_type, profile, report_dir): + """ + Returns the file name for a given file type + + :param file_type: can be one of the following: + 'RESULTS' - results JSON/JS file + 'EXCEPTIONS' - exceptions file + 'HTMLREPORT' - HTML report + 'RULESET' - ruleset file + 'ERRORS' - errors file + """ + + if file_type == 'RESULTS': + filename = DEFAULT_RESULT_FILE + first_line = 'scoutsuite_results =' + elif file_type == 'EXCEPTIONS': + filename = DEFAULT_EXCEPTIONS_FILE + first_line = 'exceptions =' + elif file_type == 'HTMLREPORT': + filename = DEFAULT_HTMLREPORT_FILE + first_line = None + elif file_type == 'RULESET': + filename = DEFAULT_RULESET_FILE + first_line = 'scoutsuite_results =' + elif file_type == 'ERRORS': + filename = DEFAULT_ERRORS_FILE + first_line = None + else: + print_error('Invalid file type provided: {}'.format(file_type)) + raise Exception + + # Append profile name if necessary + if profile != 'default' and file_type != 'RULESET': + name, extention = filename.split('.') + filename = '%s-%s.%s' % (name, profile, extention) + + return (os.path.join(report_dir, filename), first_line) + From d2641fe6c84577f4bc8f2b9bb611ef9176bb6afa Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Tue, 26 Mar 2019 10:12:12 +0100 Subject: [PATCH 577/667] Fix errors --- ScoutSuite/core/console.py | 20 ++++++++++++++------ ScoutSuite/providers/base/provider.py | 2 +- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/ScoutSuite/core/console.py b/ScoutSuite/core/console.py index daf7c51c7..b09ea2adf 100644 --- a/ScoutSuite/core/console.py +++ b/ScoutSuite/core/console.py @@ -42,20 +42,28 @@ def print_error(msg): def print_exception(exception, additional_details=None): - exc_type, exc_obj, exc_tb = sys.exc_info() - fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] + try: + exc_type, exc_obj, exc_tb = sys.exc_info() + file_name = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] + line_number = exc_tb.tb_lineno + traceback_exc = traceback.format_exc() + str = '{} L{}: {}'.format(file_name, line_number, exception) + except Exception as e: + file_name = None + line_number = None + traceback_exc = None + str = '{}'.format(exception) - str = '{} L{}: {}'.format(fname, exc_tb.tb_lineno, exception) global verbose_exceptions if verbose_exceptions: logger.exception(str) else: logger.error(str) - ERRORS_LIST.append({'file': fname, - 'line': exc_tb.tb_lineno, + ERRORS_LIST.append({'file': file_name, + 'line': line_number, 'exception': '{}'.format(exception), - 'traceback': '{}'.format(traceback.format_exc()), + 'traceback': '{}'.format(traceback_exc), 'additional_details': additional_details}) diff --git a/ScoutSuite/providers/base/provider.py b/ScoutSuite/providers/base/provider.py index f1a89b4bc..7260fda14 100644 --- a/ScoutSuite/providers/base/provider.py +++ b/ScoutSuite/providers/base/provider.py @@ -119,7 +119,7 @@ def _build_services_list(supported_services, services, skipped_services): print_exception('Service \"{}\" does not exist, skipping.'.format(service)) error = True if error: - print_exception('Available services are: {}'.format(str(list(supported_services)).strip('[]'))) + print_info('Available services are: {}'.format(str(list(supported_services)).strip('[]'))) return [s for s in supported_services if (services == [] or s in services) and s not in skipped_services] From 7c3ac627fe26c0cc66c82454c9604769f1c0ccfd Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 26 Mar 2019 08:58:20 -0400 Subject: [PATCH 578/667] Removed SQS old implementation --- ScoutSuite/providers/aws/services/sqs.py | 55 ------------------------ 1 file changed, 55 deletions(-) delete mode 100644 ScoutSuite/providers/aws/services/sqs.py diff --git a/ScoutSuite/providers/aws/services/sqs.py b/ScoutSuite/providers/aws/services/sqs.py deleted file mode 100644 index 11f7f1fde..000000000 --- a/ScoutSuite/providers/aws/services/sqs.py +++ /dev/null @@ -1,55 +0,0 @@ -# -*- coding: utf-8 -*- - -import json - -from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients - - -######################################## -# SQSRegionConfig -######################################## - -class SQSRegionConfig(RegionConfig): - """ - SQS configuration for a single AWS region - """ - queues = {} - - def parse_queue(self, global_params, region, queue_url): - """ - Parse a single queue and fetch additional attributes - - :param global_params: Parameters shared for all regions - :param region: Name of the AWS region - :param queue_url: URL of the AWS queue - """ - queue = {'QueueUrl': queue_url} - attributes = api_clients[region].get_queue_attributes(QueueUrl=queue_url, - AttributeNames=['CreatedTimestamp', 'Policy', 'QueueArn', - 'KmsMasterKeyId'])['Attributes'] - queue['arn'] = attributes.pop('QueueArn') - queue['kms_master_key_id'] = attributes.pop('KmsMasterKeyId', None) - queue['CreatedTimestamp'] = attributes.pop('CreatedTimestamp', None) - - if 'Policy' in attributes: - queue['Policy'] = json.loads(attributes['Policy']) - else: - queue['Policy'] = {'Statement': []} - - queue['name'] = queue['arn'].split(':')[-1] - self.queues[queue['name']] = queue - - -######################################## -# SQSConfig -######################################## - -class SQSConfig(RegionalServiceConfig): - """ - SQS configuration for all AWS regions - """ - - region_config_class = SQSRegionConfig - - def __init__(self, service_metadata, thread_config=4): - super(SQSConfig, self).__init__(service_metadata, thread_config) From 98eb298dd34b3f62248383a142a10fa707f5daf0 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 26 Mar 2019 09:07:50 -0400 Subject: [PATCH 579/667] Cleaned metadata.json --- ScoutSuite/providers/aws/metadata.json | 52 -------------------------- 1 file changed, 52 deletions(-) diff --git a/ScoutSuite/providers/aws/metadata.json b/ScoutSuite/providers/aws/metadata.json index 463969935..67c9f58a9 100644 --- a/ScoutSuite/providers/aws/metadata.json +++ b/ScoutSuite/providers/aws/metadata.json @@ -3,8 +3,6 @@ "emr": { "resources": { "clusters": { - "api_call": "list_clusters", - "response": "Clusters", "cols": 2, "path": "services.emr.regions.id.vpcs.id.clusters", "callbacks": [ @@ -19,8 +17,6 @@ "cloudformation": { "resources": { "stacks": { - "api_call": "describe_stacks", - "response": "Stacks", "cols": 2, "path": "services.cloudformation.regions.id.stacks", "callbacks": [ @@ -36,8 +32,6 @@ "path": "services.cloudtrail.regions" }, "trails": { - "api_call": "describe_trails", - "response": "trailList", "cols": 2, "path": "services.cloudtrail.regions.id.trails" } @@ -46,8 +40,6 @@ "cloudwatch": { "resources": { "alarms": { - "api_call": "describe_alarms", - "response": "MetricAlarms", "cols": 2, "path": "services.cloudwatch.regions.id.alarms" } @@ -66,8 +58,6 @@ "path": "services.config.regions" }, "recorders": { - "api_call": "describe_configuration_recorders", - "response": "ConfigurationRecorders", "path": "services.config.regions.id.recorders" } } @@ -139,8 +129,6 @@ "directconnect": { "resources": { "connections": { - "api_call": "describe_connections", - "response": "connections", "cols": 2, "path": "services.directconnect.connections" } @@ -160,8 +148,6 @@ "ec2": { "resources": { "instances": { - "api_call": "describe_instances", - "response": "Reservations", "cols": 2, "path": "services.ec2.regions.id.vpcs.id.instances", "callbacks": [ @@ -169,29 +155,18 @@ ] }, "security_groups": { - "api_call": "describe_security_groups", - "response": "SecurityGroups", "cols": 2, "path": "services.ec2.regions.id.vpcs.id.security_groups" }, "volumes": { - "api_call": "describe_volumes", - "response": "Volumes", "cols": 2, "path": "services.ec2.regions.id.volumes" }, "snapshots": { - "api_call": "describe_snapshots", - "params": { - "OwnerIds": [ "_AWS_ACCOUNT_ID_" ] - }, - "response": "Snapshots", "cols": 2, "path": "services.ec2.regions.id.snapshots" }, "network_interfaces": { - "api_call": "describe_network_interfaces", - "response": "NetworkInterfaces", "path": "services.ec2.regions.id.vpcs.id.network_interfaces", "hidden": true, "callbacks": [ @@ -199,11 +174,6 @@ ] }, "images": { - "api_call": "describe_images", - "params": { - "Owners": [ "_AWS_ACCOUNT_ID_" ] - }, - "response": "Images", "cols": 2, "path": "services.ec2.regions.id.images" } @@ -257,8 +227,6 @@ "awslambda": { "resources": { "functions": { - "api_call": "list_functions", - "response": "Functions", "path": "services.awslambda.regions.id.functions", "callbacks": [ [ "match_security_groups_and_resources_callback", {"status_path": ["Runtime"], "sg_list_attribute_name": [ "VpcConfig", "SecurityGroupIds" ]} ] @@ -323,8 +291,6 @@ "dynamodb": { "resources": { "tables": { - "api_call": "list_tables", - "response": "TableNames", "path": "services.dynamodb.regions.id.tables" } } @@ -332,8 +298,6 @@ "elasticache": { "resources": { "clusters": { - "api_call": "describe_cache_clusters", - "response": "CacheClusters", "cols": 2, "path": "services.elasticache.regions.id.vpcs.id.clusters", "callbacks": [ @@ -342,14 +306,10 @@ ] }, "parameter_groups": { - "api_to_call": "describe_db_parameter_groups", - "response": "DBParameterGroups", "cols": 2, "path": "services.elasticache.regions.id.parameter_groups" }, "security_groups": { - "api_call": "describe_cache_security_groups", - "response": "CacheSecurityGroups", "no_exceptions": true, "cols": 2, "path": "services.elasticache.regions.id.security_groups" @@ -363,8 +323,6 @@ "rds": { "resources": { "instances": { - "api_call": "describe_db_instances", - "response": "DBInstances", "cols": 2, "path": "services.rds.regions.id.vpcs.id.instances", "callbacks": [ @@ -373,27 +331,19 @@ ] }, "security_groups": { - "api_call": "describe_db_security_groups", - "response": "DBSecurityGroups", "no_exceptions": true, "cols": 2, "path": "services.rds.regions.id.security_groups" }, "snapshots": { - "api_call": "describe_db_snapshots", - "response": "DBSnapshots", "cols": 2, "path": "services.rds.regions.id.vpcs.id.snapshots" }, "parameter_groups": { - "api_call": "describe_db_parameter_groups", - "response": "DBParameterGroups", "cols": 2, "path": "services.rds.regions.id.parameter_groups" }, "subnet_groups": { - "api_call": "describe_db_subnet_groups", - "response": "DBSubnetGroups", "cols": 2, "path": "services.rds.regions.id.vpcs.id.subnet_groups" } @@ -437,8 +387,6 @@ "hidden": true, "resources": { "file_systems": { - "api_call": "describe_file_systems", - "response": "FileSystems", "path": "services.efs.regions.id.file_systems", "callbacks": [ [ "match_security_groups_and_resources_callback", {"status_path": ["LifeCycleState"], "sg_list_attribute_name": ["security_groups"]} ] From 0458702bcffe924805073e2a76374032e17b7b67 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 26 Mar 2019 09:08:04 -0400 Subject: [PATCH 580/667] Added basic logging --- ScoutSuite/providers/base/configs/services.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/providers/base/configs/services.py b/ScoutSuite/providers/base/configs/services.py index 1feea98f2..8b8f5df94 100644 --- a/ScoutSuite/providers/base/configs/services.py +++ b/ScoutSuite/providers/base/configs/services.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from ScoutSuite.core.console import print_error, print_exception, print_debug +from ScoutSuite.core.console import print_error, print_exception, print_debug, print_info from ScoutSuite.providers.aws.utils import get_partition_name @@ -17,6 +17,7 @@ async def fetch(self, credentials, services=None, regions=None): regions = [] if regions is None else regions for service in vars(self): try: + print_info('Fetching ' + service + '.') # skip services if services != [] and service not in services: continue From b0e8f5f886fc49ea4cd000b70e6cbab1495f646a Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 26 Mar 2019 09:20:52 -0400 Subject: [PATCH 581/667] Fixed missing attribute in stack formation stacks --- ScoutSuite/providers/aws/resources/cloudformation/service.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ScoutSuite/providers/aws/resources/cloudformation/service.py b/ScoutSuite/providers/aws/resources/cloudformation/service.py index 234db3475..f4c395d47 100644 --- a/ScoutSuite/providers/aws/resources/cloudformation/service.py +++ b/ScoutSuite/providers/aws/resources/cloudformation/service.py @@ -16,6 +16,7 @@ def _parse_stack(self, raw_stack): raw_stack['id'] = raw_stack.pop('StackId') raw_stack['name'] = raw_stack.pop('StackName') raw_stack['drifted'] = raw_stack.pop('DriftInformation')['StackDriftStatus'] == 'DRIFTED' + raw_stack['termination_protection'] = raw_stack['EnableTerminationProtection'] template = raw_stack.pop('template') raw_stack['deletion_policy'] = self.has_deletion_policy(template) From 2a97948ce67a35574baa4b7c81ae72ce633b2f62 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 26 Mar 2019 09:36:47 -0400 Subject: [PATCH 582/667] Fixed weird merge --- ScoutSuite/providers/aws/facade/facade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/facade/facade.py b/ScoutSuite/providers/aws/facade/facade.py index 49c7fd75a..c6049fc6b 100644 --- a/ScoutSuite/providers/aws/facade/facade.py +++ b/ScoutSuite/providers/aws/facade/facade.py @@ -33,7 +33,7 @@ pass -class AWSFacade(AWSaacade): +class AWSFacade(AWSBaseFacade): def __init__(self, credentials: dict = None): self._set_session(credentials) From 3d95deec684acbf8623b195935a581bb88bf39a4 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 26 Mar 2019 09:51:03 -0400 Subject: [PATCH 583/667] Removed old import --- ScoutSuite/providers/aws/configs/services.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index 6c82b7e8b..e43c0895c 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -16,7 +16,6 @@ from ScoutSuite.providers.aws.resources.rds.service import RDS from ScoutSuite.providers.aws.resources.redshift.service import Redshift from ScoutSuite.providers.aws.resources.s3.service import S3 -from ScoutSuite.providers.aws.services.s3 import S3Config from ScoutSuite.providers.aws.resources.vpc.service import VPC from ScoutSuite.providers.aws.resources.sqs.service import SQS from ScoutSuite.providers.aws.resources.ses.service import SES From 34b97622b2e2ca055b4c305365b1eaeda1193b91 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 26 Mar 2019 10:32:12 -0400 Subject: [PATCH 584/667] Improved logging --- ScoutSuite/providers/base/configs/services.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/providers/base/configs/services.py b/ScoutSuite/providers/base/configs/services.py index 8b8f5df94..8173dc937 100644 --- a/ScoutSuite/providers/base/configs/services.py +++ b/ScoutSuite/providers/base/configs/services.py @@ -17,10 +17,12 @@ async def fetch(self, credentials, services=None, regions=None): regions = [] if regions is None else regions for service in vars(self): try: - print_info('Fetching ' + service + '.') # skip services if services != [] and service not in services: + print_debug('Skipping ' + service + '.') continue + + print_info('Fetching ' + service + '.') service_config = getattr(self, service) # call fetch method for the service if 'fetch_all' in dir(service_config): From 4e0831bd2f9bf0dd99a45fe3940897ce4c227cf0 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 26 Mar 2019 10:34:58 -0400 Subject: [PATCH 585/667] Implemented tasks parallelizing util --- ScoutSuite/providers/aws/facade/iam.py | 51 +++++++++++++------------- ScoutSuite/providers/utils.py | 16 ++++++++ 2 files changed, 41 insertions(+), 26 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/iam.py b/ScoutSuite/providers/aws/facade/iam.py index 95420b71e..08e1ccd03 100644 --- a/ScoutSuite/providers/aws/facade/iam.py +++ b/ScoutSuite/providers/aws/facade/iam.py @@ -3,7 +3,7 @@ from ScoutSuite.core.console import print_error, print_exception from ScoutSuite.providers.utils import run_concurrently, get_non_provider_id from ScoutSuite.providers.aws.utils import is_throttled -from ScoutSuite.providers.utils import run_concurrently +from ScoutSuite.providers.utils import run_concurrently, run_tasks_concurrently class IAMFacade(AWSBaseFacade): @@ -46,32 +46,31 @@ async def get_groups(self): async def get_policies(self): policies = await AWSFacadeUtils.get_all_pages('iam', None, self.session, 'list_policies', 'Policies', OnlyAttached=True) - client = AWSFacadeUtils.get_client('iam', self.session) + return await run_tasks_concurrently({ (lambda: self.get_detailed_policy(policy) for policy in policies)} ) - # TODO: Parallelize this - for policy in policies: - policy_version = await run_concurrently( - lambda: client.get_policy_version(PolicyArn=policy['Arn'], VersionId=policy['DefaultVersionId'])) - policy['PolicyDocument'] = policy_version['PolicyVersion']['Document'] - - policy['attached_to'] = {} - attached_entities = await AWSFacadeUtils.get_multiple_entities_from_all_pages('iam', None, self.session, 'list_entities_for_policy', ['PolicyGroups', 'PolicyRoles', 'PolicyUsers'], PolicyArn=policy['Arn']) - - for entity_type in attached_entities: - resource_type = entity_type.replace('Policy', '').lower() - if len(attached_entities[entity_type]): - policy['attached_to'][resource_type] = [] - - for entity in attached_entities[entity_type]: - name_field = entity_type.replace('Policy', '')[ - :-1] + 'Name' - resource_name = entity[name_field] - id_field = entity_type.replace('Policy', '')[:-1] + 'Id' - resource_id = entity[id_field] - policy['attached_to'][resource_type].append( - {'name': resource_name, 'id': resource_id}) - - return policies + async def get_detailed_policy(self, policy): + client = AWSFacadeUtils.get_client('iam', self.session) + policy_version = await run_concurrently(lambda: client.get_policy_version(PolicyArn=policy['Arn'], VersionId=policy['DefaultVersionId'])) + policy['PolicyDocument'] = policy_version['PolicyVersion']['Document'] + + policy['attached_to'] = {} + attached_entities = await AWSFacadeUtils.get_multiple_entities_from_all_pages('iam', None, self.session, 'list_entities_for_policy', ['PolicyGroups', 'PolicyRoles', 'PolicyUsers'], PolicyArn=policy['Arn']) + + for entity_type in attached_entities: + resource_type = entity_type.replace('Policy', '').lower() + if len(attached_entities[entity_type]): + policy['attached_to'][resource_type] = [] + + for entity in attached_entities[entity_type]: + name_field = entity_type.replace('Policy', '')[ + :-1] + 'Name' + resource_name = entity[name_field] + id_field = entity_type.replace('Policy', '')[:-1] + 'Id' + resource_id = entity[id_field] + policy['attached_to'][resource_type].append( + {'name': resource_name, 'id': resource_id}) + + return policy async def get_users(self): client = AWSFacadeUtils.get_client('iam', self.session) diff --git a/ScoutSuite/providers/utils.py b/ScoutSuite/providers/utils.py index b74128f6a..96b53b806 100644 --- a/ScoutSuite/providers/utils.py +++ b/ScoutSuite/providers/utils.py @@ -17,3 +17,19 @@ def get_non_provider_id(name): def run_concurrently(func): return asyncio.get_event_loop().run_in_executor(executor=None, func=func) + +async def run_tasks_concurrently(tasks): + entities = [] + # tasks = { + # asyncio.ensure_future(run_concurrently(lambda: self._get_table(region, table_name))) for table_name in tables_names + # } + + parallelized_tasks = { + asyncio.ensure_future(run_concurrently(task)) for task in tasks + } + + for task in asyncio.as_completed(parallelized_tasks): + entity = await task + entities.append(entity) + + return entities From 34e481422711e0b56a94a2822655fd7d499c0f95 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 26 Mar 2019 11:19:59 -0400 Subject: [PATCH 586/667] Improved util --- ScoutSuite/providers/utils.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ScoutSuite/providers/utils.py b/ScoutSuite/providers/utils.py index 96b53b806..14f09db60 100644 --- a/ScoutSuite/providers/utils.py +++ b/ScoutSuite/providers/utils.py @@ -20,12 +20,9 @@ def run_concurrently(func): async def run_tasks_concurrently(tasks): entities = [] - # tasks = { - # asyncio.ensure_future(run_concurrently(lambda: self._get_table(region, table_name))) for table_name in tables_names - # } parallelized_tasks = { - asyncio.ensure_future(run_concurrently(task)) for task in tasks + asyncio.ensure_future(task) for task in tasks } for task in asyncio.as_completed(parallelized_tasks): From eb0e048b5ede6f890b542de9c14c13b7c5afa15c Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 26 Mar 2019 11:20:25 -0400 Subject: [PATCH 587/667] Integrated util with EMR --- ScoutSuite/providers/aws/facade/emr.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/emr.py b/ScoutSuite/providers/aws/facade/emr.py index ebeb6b448..7acadced6 100644 --- a/ScoutSuite/providers/aws/facade/emr.py +++ b/ScoutSuite/providers/aws/facade/emr.py @@ -1,6 +1,6 @@ from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade -from ScoutSuite.providers.utils import run_concurrently +from ScoutSuite.providers.utils import run_concurrently, run_tasks_concurrently import asyncio @@ -8,15 +8,8 @@ class EMRFacade(AWSBaseFacade): async def get_clusters(self, region): clusters_list = await AWSFacadeUtils.get_all_pages('emr', region, self.session, 'list_clusters', 'Clusters') client = AWSFacadeUtils.get_client('emr', self.session, region) - clusters_descriptions = [] cluster_ids = [cluster['Id'] for cluster in clusters_list] - tasks = { - asyncio.ensure_future( - run_concurrently(lambda: client.describe_cluster(ClusterId=cluster_id)['Cluster']) - ) for cluster_id in cluster_ids - } - for task in asyncio.as_completed(tasks): - cluster = await task - clusters_descriptions.append(cluster) - - return clusters_descriptions + + return await run_tasks_concurrently({ + run_concurrently(lambda: client.describe_cluster(ClusterId=id)['Cluster']) for id in cluster_ids + }) From 44c4fd4e84c973f054eed1680b04e5e6433c0a1a Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 26 Mar 2019 11:20:41 -0400 Subject: [PATCH 588/667] Addressed a few parallelizing todos --- ScoutSuite/providers/aws/facade/iam.py | 58 ++++++++++++++------------ 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/iam.py b/ScoutSuite/providers/aws/facade/iam.py index 08e1ccd03..74d623791 100644 --- a/ScoutSuite/providers/aws/facade/iam.py +++ b/ScoutSuite/providers/aws/facade/iam.py @@ -46,7 +46,9 @@ async def get_groups(self): async def get_policies(self): policies = await AWSFacadeUtils.get_all_pages('iam', None, self.session, 'list_policies', 'Policies', OnlyAttached=True) - return await run_tasks_concurrently({ (lambda: self.get_detailed_policy(policy) for policy in policies)} ) + return await run_tasks_concurrently({ + self.get_detailed_policy(policy) for policy in policies + }) async def get_detailed_policy(self, policy): client = AWSFacadeUtils.get_client('iam', self.session) @@ -73,31 +75,33 @@ async def get_detailed_policy(self, policy): return policy async def get_users(self): - client = AWSFacadeUtils.get_client('iam', self.session) users = await AWSFacadeUtils.get_all_pages('iam', None, self.session, 'list_users', 'Users') + return await run_tasks_concurrently({ + self.get_detailed_user(user) for user in users + }) - # TODO: Parallelize this - for user in users: - user_name = user['UserName'] - user_id = user['UserId'] - - policies = self._get_inline_policies('user', user_id, user_name) - if len(policies): - user['inline_policies'] = policies - user['inline_policies_count'] = len(policies) - user['groups'] = [] - groups = await AWSFacadeUtils.get_all_pages('iam', None, self.session, 'list_groups_for_user', 'Groups', UserName=user_name) - for group in groups: - user['groups'].append(group['GroupName']) - try: - user['LoginProfile'] = await run_concurrently( - lambda: client.get_login_profile(UserName=user_name)['LoginProfile']) - except Exception: - pass - user['AccessKeys'] = await self._get_user_acces_keys(user_name) - user['MFADevices'] = await self._get_user_mfa_devices(user_name) + async def get_detailed_user(self, user): + client = AWSFacadeUtils.get_client('iam', self.session) + user_name = user['UserName'] + user_id = user['UserId'] + + policies = self._get_inline_policies('user', user_id, user_name) + if len(policies): + user['inline_policies'] = policies + user['inline_policies_count'] = len(policies) + user['groups'] = [] + groups = await AWSFacadeUtils.get_all_pages('iam', None, self.session, 'list_groups_for_user', 'Groups', UserName=user_name) + for group in groups: + user['groups'].append(group['GroupName']) + try: + user['LoginProfile'] = await run_concurrently( + lambda: client.get_login_profile(UserName=user_name)['LoginProfile']) + except Exception: + pass + user['AccessKeys'] = await self._get_user_acces_keys(user_name) + user['MFADevices'] = await self._get_user_mfa_devices(user_name) - return users + return user async def get_roles(self): roles = await AWSFacadeUtils.get_all_pages('iam', None, self.session, 'list_roles', 'Roles') @@ -154,8 +158,7 @@ async def _fetch_group_users(self, group_name): users.append(user['UserId']) return users - # TODO: Make this async - def _get_inline_policies(self, iam_resource_type, resource_id, resource_name): + async def _get_inline_policies(self, iam_resource_type, resource_id, resource_name): client = AWSFacadeUtils.get_client('iam', self.session) get_policy_method = getattr( client, 'get_' + iam_resource_type + '_policy') @@ -163,14 +166,16 @@ def _get_inline_policies(self, iam_resource_type, resource_id, resource_name): list_policy_method = getattr( client, 'list_' + iam_resource_type + '_policies') args = {iam_resource_type.title() + 'Name': resource_name} + try: - policy_names = list_policy_method(**args)['PolicyNames'] + policy_names = await run_concurrently(lambda: list_policy_method(**args)['PolicyNames']) except Exception as e: if is_throttled(e): raise e else: print_exception(e) return fetched_policies + try: for policy_name in policy_names: args['PolicyName'] = policy_name @@ -186,6 +191,7 @@ def _get_inline_policies(self, iam_resource_type, resource_id, resource_name): raise e else: print_exception(e) + return fetched_policies def _normalize_statements(self, policy_document): From 4aa9a51fd7c221cb30929ca53d84ddf3e05e6347 Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Tue, 26 Mar 2019 16:36:36 +0100 Subject: [PATCH 589/667] Merge changes --- ScoutSuite/providers/base/configs/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/base/configs/services.py b/ScoutSuite/providers/base/configs/services.py index b9030bbdf..8fcf4729c 100644 --- a/ScoutSuite/providers/base/configs/services.py +++ b/ScoutSuite/providers/base/configs/services.py @@ -20,7 +20,7 @@ async def fetch(self, credentials, services=None, regions=None): try: # skip services if services != [] and service not in services: - print_debug('Skipping ' + service + '.') + print_debug('Skipping the {} service'.format(format_service_name(service))) continue print_info('Fetching resources for the {} service'.format(format_service_name(service))) service_config = getattr(self, service) From 22edc76e50eaadcab6ac0f28a9c47dafe1b5f10e Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 26 Mar 2019 11:47:06 -0400 Subject: [PATCH 590/667] Added docstrings --- ScoutSuite/providers/aws/facade/utils.py | 32 +++++++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/utils.py b/ScoutSuite/providers/aws/facade/utils.py index 3692db3e1..51b0d253c 100644 --- a/ScoutSuite/providers/aws/facade/utils.py +++ b/ScoutSuite/providers/aws/facade/utils.py @@ -3,17 +3,41 @@ from ScoutSuite.providers.utils import run_concurrently -# TODO: Add docstrings class AWSFacadeUtils: _clients = {} @staticmethod async def get_all_pages(service: str, region: str, session: boto3.session.Session, paginator_name: str, entity: str, **paginator_args): + """ + Gets all the entities from a paginator given an entity key + + :param service:str: Name of the AWS service (ec2, iam, etc.) + :param region:str: Region + :param session:boto3.session.Session: Boto3 session used to authenticate the client + :param paginator_name:str: Name of the paginator + :param entity:str: Key used to retreive the entities in the paginator's response + :param **paginator_args: Arguments passed to the paginator + + :return: A list of the fetched entities. + """ + results = await AWSFacadeUtils.get_multiple_entities_from_all_pages(service, region, session, paginator_name, [entity], **paginator_args) return results[entity] @staticmethod async def get_multiple_entities_from_all_pages(service: str, region: str, session: boto3.session.Session, paginator_name: str, entities: list, **paginator_args): + """ + Gets all the entities from a paginator given multiple entitiy keys + :param service:str: Name of the AWS service (ec2, iam, etc.) + :param region:str: Region + :param session:boto3.session.Session: Boto3 session used to authenticate the client + :param paginator_name:str: Name of the paginator + :param entities:list: Keys used to retreive the entities in the paginator's response + :param **paginator_args: Arguments passed to the paginator + + :return: A dictionary with the entity keys as keys, and the fetched entities lists as values. + """ + client = AWSFacadeUtils.get_client(service, session, region) # Building a paginator doesn't require any API call so no need to do it concurrently: @@ -39,9 +63,9 @@ def get_client(service: str, session: boto3.session.Session, region: str=None): """ Instantiates an AWS API client - :param service: Service targeted, e.g. ec2 - :param session: The aws session - :param region: Region desired, e.g. us-east-2 + :param service: Service targeted, e.g. ec2 + :param session: The aws session + :param region: Region desired, e.g. us-east-2 :return: """ From 455082d6da01c6957ccaf744516f52c5bffcd80a Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 26 Mar 2019 11:56:50 -0400 Subject: [PATCH 591/667] Made get client thread-safer --- ScoutSuite/providers/aws/facade/utils.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/utils.py b/ScoutSuite/providers/aws/facade/utils.py index 51b0d253c..4649d66b3 100644 --- a/ScoutSuite/providers/aws/facade/utils.py +++ b/ScoutSuite/providers/aws/facade/utils.py @@ -37,7 +37,7 @@ async def get_multiple_entities_from_all_pages(service: str, region: str, sessio :return: A dictionary with the entity keys as keys, and the fetched entities lists as values. """ - + client = AWSFacadeUtils.get_client(service, session, region) # Building a paginator doesn't require any API call so no need to do it concurrently: @@ -59,7 +59,7 @@ def _get_all_pages_from_paginator(paginator, entities: list): return resources @staticmethod - def get_client(service: str, session: boto3.session.Session, region: str=None): + def get_client(service: str, session: boto3.session.Session, region: str = None): """ Instantiates an AWS API client @@ -69,7 +69,5 @@ def get_client(service: str, session: boto3.session.Session, region: str=None): :return: """ - - # TODO: investigate the use of a mutex to avoid useless creation of a same type of client among threads: - client = session.client(service, region_name=region) if region else session.client(service) - return AWSFacadeUtils._clients.setdefault((service, region), client) + return AWSFacadeUtils._clients.setdefault((service, region), + session.client(service, region_name=region) if region else session.client(service)) From 66b3544bf7c7481ed0e800cb87831ebe023e8ba8 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 26 Mar 2019 13:48:35 -0400 Subject: [PATCH 592/667] Awaited get_inline_policies --- ScoutSuite/providers/aws/facade/iam.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/iam.py b/ScoutSuite/providers/aws/facade/iam.py index 74d623791..7b1cbd2e6 100644 --- a/ScoutSuite/providers/aws/facade/iam.py +++ b/ScoutSuite/providers/aws/facade/iam.py @@ -37,7 +37,7 @@ async def get_groups(self): groups = await AWSFacadeUtils.get_all_pages('iam', None, self.session, 'list_groups', 'Groups') for group in groups: group['Users'] = await self._fetch_group_users(group['GroupName']) - policies = self._get_inline_policies( + policies = await self._get_inline_policies( 'group', group['GroupId'], group['GroupName']) if len(policies): group['inline_policies'] = policies @@ -85,7 +85,7 @@ async def get_detailed_user(self, user): user_name = user['UserName'] user_id = user['UserId'] - policies = self._get_inline_policies('user', user_id, user_name) + policies = await self._get_inline_policies('user', user_id, user_name) if len(policies): user['inline_policies'] = policies user['inline_policies_count'] = len(policies) @@ -111,7 +111,7 @@ async def get_roles(self): role['instances_count'] = 'N/A' # Get role policies - policies = self._get_inline_policies( + policies = await self._get_inline_policies( 'role', role['RoleId'], role['RoleName']) if len(policies): role['inline_policies'] = policies From c60a67d39cf7f2e158afaa84971de31abab9e8ec Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 26 Mar 2019 13:55:24 -0400 Subject: [PATCH 593/667] Parallelized loadbalancers fetching --- .../aws/resources/elbv2/load_balancers.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/elbv2/load_balancers.py b/ScoutSuite/providers/aws/resources/elbv2/load_balancers.py index e67ca7f5a..41ba05676 100644 --- a/ScoutSuite/providers/aws/resources/elbv2/load_balancers.py +++ b/ScoutSuite/providers/aws/resources/elbv2/load_balancers.py @@ -1,6 +1,7 @@ +import asyncio + from ScoutSuite.providers.aws.resources.resources import AWSCompositeResources from ScoutSuite.providers.utils import get_non_provider_id - from .listeners import Listeners @@ -11,12 +12,13 @@ class LoadBalancers(AWSCompositeResources): async def fetch_all(self, **kwargs): raw_loads_balancers = await self.facade.elbv2.get_load_balancers(self.scope['region'], self.scope['vpc']) - # TODO: parallelize the following loop which is async: - for raw_load_balancer in raw_loads_balancers: - id, load_balancer = await self._parse_load_balancer(raw_load_balancer) - self[id] = load_balancer - await self._fetch_children( - parent=load_balancer, scope={'region': self.scope['region'], 'load_balancer_arn': load_balancer['arn']}) + await asyncio.wait({asyncio.ensure_future(self._fetch_one(raw_load_balancer)) for raw_load_balancer in raw_loads_balancers}) + + async def _fetch_one(self, raw_load_balancer): + id, load_balancer = await self._parse_load_balancer(raw_load_balancer) + self[id] = load_balancer + await self._fetch_children(parent=load_balancer, scope={'region': self.scope['region'], 'load_balancer_arn': load_balancer['arn']}) + async def _parse_load_balancer(self, load_balancer): load_balancer['arn'] = load_balancer.pop('LoadBalancerArn') From 45de0626231774885443c2fdb163b47d4a66619f Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 26 Mar 2019 19:26:04 -0400 Subject: [PATCH 594/667] Deleted unused files --- ScoutSuite/providers/aws/configs/base.py | 43 --- ScoutSuite/providers/aws/configs/regions.py | 340 -------------------- ScoutSuite/providers/aws/configs/vpc.py | 15 - 3 files changed, 398 deletions(-) delete mode 100644 ScoutSuite/providers/aws/configs/base.py delete mode 100644 ScoutSuite/providers/aws/configs/regions.py delete mode 100644 ScoutSuite/providers/aws/configs/vpc.py diff --git a/ScoutSuite/providers/aws/configs/base.py b/ScoutSuite/providers/aws/configs/base.py deleted file mode 100644 index 24d5d09ca..000000000 --- a/ScoutSuite/providers/aws/configs/base.py +++ /dev/null @@ -1,43 +0,0 @@ -# -*- coding: utf-8 -*- - -# Python2 vs Python3 -try: - from Queue import Queue -except ImportError: - from queue import Queue - -from ScoutSuite.providers.aws.utils import handle_truncated_response -from ScoutSuite.providers.base.configs.base import BaseConfig - - -class AWSBaseConfig(BaseConfig): - - def _is_provider(self, provider_name): - return provider_name == 'aws' - - def _get_method(self, api_client, target_type, list_method_name): - """ - Gets the appropriate method, required as each provider may have particularities - - :return: - """ - - method = getattr(api_client, list_method_name) - - return method - - def _get_targets(self, response_attribute, api_client, method, list_params, ignore_list_error): - """ - Fetch the targets, required as each provider may have particularities - - :return: - """ - - if type(list_params) != list: - list_params = [list_params] - - targets = [] - for lp in list_params: - targets += handle_truncated_response(method, lp, [response_attribute])[response_attribute] - - return targets diff --git a/ScoutSuite/providers/aws/configs/regions.py b/ScoutSuite/providers/aws/configs/regions.py deleted file mode 100644 index 254145635..000000000 --- a/ScoutSuite/providers/aws/configs/regions.py +++ /dev/null @@ -1,340 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Base classes and functions for region-specific services -""" - -import copy -import re -from threading import Thread - -# Python2 vs Python3 -try: - from Queue import Queue -except ImportError: - from queue import Queue - -from ScoutSuite.core.console import print_exception, print_info - -from ScoutSuite.providers.base.configs import resource_id_map -from ScoutSuite.providers.base.configs.threads import thread_configs -from ScoutSuite.providers.aws.configs.vpc import VPCConfig -from ScoutSuite.utils import format_service_name, manage_dictionary -from ScoutSuite.providers.aws.utils import is_throttled, build_region_list, connect_service, get_name, \ - get_aws_account_id, handle_truncated_response -from ScoutSuite.providers.aws.configs.base import BaseConfig -from ScoutSuite.output.console import FetchStatusLogger - -######################################## -# Globals -######################################## - -api_clients = dict() - -first_cap_re = re.compile('(.)([A-Z][a-z]+)') -all_caps_re = re.compile('([a-z0-9])([A-Z])') - - -######################################## -# RegionalServiceConfig -######################################## - - -class RegionalServiceConfig(object): - """ - Single service configuration for non-global services - - :ivar regions: Dictionary of regions - :ivar service: Name of the service - """ - - def __init__(self, service_metadata=None, thread_config=4): - service_metadata = {} if service_metadata is None else service_metadata - - self.regions = {} - self.thread_config = thread_configs[thread_config] - self.service = re.sub(r'Config$', "", type(self).__name__).lower() - self.fetchstatuslogger = None - - # Booleans that define if threads should keep running - self.run_q_threads = True - self.run_qr_threads = True - - if service_metadata != {}: - self.resource_types = {'global': [], 'region': [], 'vpc': []} - self.targets = {'first_region': (), 'other_regions': ()} - for resource in service_metadata['resources']: - only_first_region = False - if re.match(r'.*?\.vpcs\.id\..*?', service_metadata['resources'][resource]['path']): - self.resource_types['vpc'].append(resource) - elif re.match(r'.*?\.regions\.id\..*?', service_metadata['resources'][resource]['path']): - self.resource_types['region'].append(resource) - else: - only_first_region = True - self.resource_types['global'].append(resource) - resource_metadata = service_metadata['resources'][resource] - if 'api_call' not in resource_metadata: - continue - params = resource_metadata['params'] if 'params' in resource_metadata else {} - ignore_exceptions = resource_metadata.get('no_exceptions', False) - if not only_first_region: - self.targets['other_regions'] += ((resource, - resource_metadata['response'], - resource_metadata['api_call'], - params, - ignore_exceptions),) - self.targets['first_region'] += ((resource, - resource_metadata['response'], - resource_metadata['api_call'], - params, - ignore_exceptions),) - - def init_region_config(self, region): - """ - Initialize the region's configuration - - :param region: Name of the region - """ - self.regions[region] = self.region_config_class(region_name=region, resource_types=self.resource_types) - - def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): - """ - Fetch all the configuration supported by Scout2 for a given service - - :param credentials: F - :param regions: Name of regions to fetch data from - :param partition_name: AWS partition to connect to - :param targets: Type of resources to be fetched; defaults to all. - - """ - # Initialize targets - # Tweak params - regions = [] if regions is None else regions - realtargets = () - if not targets: - targets = self.targets - for i, target in enumerate(targets['first_region']): - params = self.tweak_params(target[3], credentials) - realtargets = realtargets + ((target[0], target[1], target[2], params, target[4]),) - targets['first_region'] = realtargets - realtargets = () - for i, target in enumerate(targets['other_regions']): - params = self.tweak_params(target[3], credentials) - realtargets = realtargets + ((target[0], target[1], target[2], params, target[4]),) - targets['other_regions'] = realtargets - - print_info('Fetching %s config...' % format_service_name(self.service)) - self.fetchstatuslogger = FetchStatusLogger(targets['first_region'], True) - api_service = 'ec2' if self.service.lower() == 'vpc' else self.service.lower() - - # Init regions - regions = build_region_list(api_service, regions, partition_name) # TODO: move this code within this class - self.fetchstatuslogger.counts['regions']['discovered'] = len(regions) - - # Threading to fetch & parse resources (queue consumer) - q = self._init_threading(self._fetch_target, {}, self.thread_config['parse']) - - # Threading to list resources (queue feeder) - qr = self._init_threading(self._fetch_region, - {'api_service': api_service, 'credentials': credentials, 'q': q, 'targets': ()}, - self.thread_config['list']) - - # Go - for i, region in enumerate(regions): - qr.put((region, targets['first_region'] if i == 0 else targets['other_regions'])) - - # Blocks until all items in the queue have been gotten and processed. - qr.join() - q.join() - - # Show completion and force newline - self.fetchstatuslogger.show(True) - - # Threads should stop running as queues are empty - self.run_qr_threads = False - self.run_q_threads = False - # Put x items in the queues to ensure threads run one last time (and exit) - for i in range(self.thread_config['parse']): - q.put(None) - for j in range(self.thread_config['list']): - qr.put(None) - - @staticmethod - def _init_threading(function, params=None, num_threads=10): - """ - Initialize queue and threads - - :param function: - :param params: - :param num_threads: - :return: - """ - params = {} if params is None else params - q = Queue(maxsize=0) # TODO: find something appropriate - for i in range(num_threads): - worker = Thread(target=function, args=(q, params)) - worker.setDaemon(True) - worker.start() - return q - - def _fetch_region(self, q, params): - global api_clients - try: - while self.run_qr_threads: - try: - region, targets = q.get() or (None, None) - if region: - self.init_region_config(region) - api_client = connect_service(params['api_service'], params['credentials'], region, silent=True) - api_clients[region] = api_client - # TODO : something here for single_region stuff - self.regions[region].fetch_all(api_client, self.fetchstatuslogger, params['q'], - targets) # params['targets']) - self.fetchstatuslogger.counts['regions']['fetched'] += 1 - except Exception as e: - print_exception(e) - finally: - q.task_done() - except Exception as e: - print_exception(e) - pass - - def _fetch_target(self, q, params): - try: - while self.run_q_threads: - try: - method, region, target = q.get() or (None, None, None) - if method: - backup = copy.deepcopy(target) - - if method.__name__ == 'store_target': - target_type = target['scout2_target_type'] - else: - target_type = method.__name__.replace('parse_', '') + 's' - method(params, region, target) - self.fetchstatuslogger.counts[target_type]['fetched'] += 1 - self.fetchstatuslogger.show() - except Exception as e: - if is_throttled(e): - q.put((method, region, backup)) - else: - print_exception(e) - finally: - q.task_done() - except Exception as e: - print_exception(e) - pass - - def finalize(self): - for t in self.fetchstatuslogger.counts: - setattr(self, '%s_count' % t, self.fetchstatuslogger.counts[t]['fetched']) - delattr(self, 'fetchstatuslogger') - for r in self.regions: - if hasattr(self.regions[r], 'fetchstatuslogger'): - delattr(self.regions[r], 'fetchstatuslogger') - - def tweak_params(self, params, credentials): - if type(params) == dict: - for k in params: - params[k] = self.tweak_params(params[k], credentials) - elif type(params) == list: - newparams = [] - for v in params: - newparams.append(self.tweak_params(v, credentials)) - params = newparams - else: - if params == '_AWS_ACCOUNT_ID_': - params = get_aws_account_id(credentials) - return params - - -######################################## -# RegionConfig -######################################## - - -# noinspection PyBroadException -class RegionConfig(BaseConfig): - """ - Base class for ... - """ - - def __init__(self, region_name, resource_types=None, **kwargs): - super(RegionConfig, self).__init__(**kwargs) - resource_types = {} if resource_types is None else resource_types - self.region = region_name - self.name = region_name - self.id = region_name - self.fetchstatuslogger = None - for resource_type in resource_types['region'] + resource_types['global']: - setattr(self, resource_type, {}) - setattr(self, '%s_count' % resource_type, 0) - if len(resource_types['vpc']) > 0: - setattr(self, 'vpcs', {}) - self.vpc_resource_types = resource_types['vpc'] - - def fetch_all(self, api_client, fetchstatuslogger, q, targets): - """ - Make all API calls as defined in metadata.json - - :param api_client: - :param fetchstatuslogger: - :param q: - :param targets: - :return: - """ - self.fetchstatuslogger = fetchstatuslogger - if targets is not None: - # Ensure targets is a tuple - if type(targets) != list and type(targets) != tuple: - targets = tuple(targets, ) - elif type(targets) != tuple: - targets = tuple(targets) - for target in targets: - self._fetch_targets(api_client, q, target) - - def _fetch_targets(self, api_client, q, target): - """ - Make an API call defined in metadata.json. - Parse the returned object as implemented in the "parse_[object name]" method. - - :param api_client: - :param q: - :param target: - :return: - """ - # Handle & format the target type - target_type, response_attribute, list_method_name, list_params, ignore_list_error = target - list_method = getattr(api_client, list_method_name) - try: - targets = handle_truncated_response(list_method, list_params, [response_attribute])[response_attribute] - except Exception as e: - if not ignore_list_error: - print_exception(e) - targets = [] - setattr(self, '%s_count' % target_type, len(targets)) - self.fetchstatuslogger.counts[target_type]['discovered'] += len(targets) - region = api_client._client_config.region_name - # Queue resources - for target in targets: - # call callback methods - try: - callback = getattr(self, 'parse_%s' % target_type[0:-1]) - except: - callback = self.store_target - target['scout2_target_type'] = target_type - if q: - # Add to the queue - q.put((callback, region, target)) - - def store_target(self, global_params, region, target): - target_type = target.pop('scout2_target_type') - if 'VpcId' in target: - vpc_id = target.pop('VpcId') - manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types)) - tmp = getattr(self, 'vpcs')[vpc_id] - target_dict = getattr(tmp, target_type) - else: - target_dict = getattr(self, target_type) - target_id = target[resource_id_map[target_type]] - get_name(target, target, resource_id_map[target_type]) - target_dict[target_id] = target diff --git a/ScoutSuite/providers/aws/configs/vpc.py b/ScoutSuite/providers/aws/configs/vpc.py deleted file mode 100644 index ccc0aa288..000000000 --- a/ScoutSuite/providers/aws/configs/vpc.py +++ /dev/null @@ -1,15 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Base class for vpc-specific services -""" - - -class VPCConfig(object): - """ - Configuration for a single VPC in a single service - """ - - def __init__(self, vpc_resource_types, name=None): - self.name = name - for resource_type in vpc_resource_types: - setattr(self, resource_type, {}) From a74bfd5f42e2b2955ac022eff457d74da1f4bc11 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 26 Mar 2019 19:46:17 -0400 Subject: [PATCH 595/667] Renamed "RegionalLambdas" to "Functions" --- ScoutSuite/providers/aws/resources/awslambda/service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/awslambda/service.py b/ScoutSuite/providers/aws/resources/awslambda/service.py index c81d15d8d..cab77586d 100644 --- a/ScoutSuite/providers/aws/resources/awslambda/service.py +++ b/ScoutSuite/providers/aws/resources/awslambda/service.py @@ -3,7 +3,7 @@ from ScoutSuite.providers.aws.resources.resources import AWSResources -class RegionalLambdas(AWSResources): +class Functions(AWSResources): async def fetch_all(self, **kwargs): raw_functions = await self.facade.awslambda.get_functions(self.scope['region']) for raw_function in raw_functions: @@ -17,7 +17,7 @@ def _parse_function(self, raw_function): class Lambdas(Regions): _children = [ - (RegionalLambdas, 'functions') + (Functions, 'functions') ] def __init__(self): From 2a035cffa9aa506ac2daca175be9f41b617cd3a6 Mon Sep 17 00:00:00 2001 From: misg Date: Wed, 27 Mar 2019 00:07:48 -0400 Subject: [PATCH 596/667] Fetch activity logs. --- .../providers/azure/facade/storageaccounts.py | 43 ++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/providers/azure/facade/storageaccounts.py b/ScoutSuite/providers/azure/facade/storageaccounts.py index ee1fc19c1..096cc4842 100644 --- a/ScoutSuite/providers/azure/facade/storageaccounts.py +++ b/ScoutSuite/providers/azure/facade/storageaccounts.py @@ -1,15 +1,56 @@ +import asyncio +import datetime + from azure.mgmt.storage import StorageManagementClient +from azure.mgmt.monitor import MonitorManagementClient from ScoutSuite.providers.utils import run_concurrently class StorageAccountsFacade: def __init__(self, credentials, subscription_id): + self._credentials = credentials + self._subscription_id = subscription_id self._client = StorageManagementClient(credentials, subscription_id) async def get_storage_accounts(self): - return await run_concurrently(self._client.storage_accounts.list) + storage_accounts = await run_concurrently( + lambda: list(self._client.storage_accounts.list()) + ) + + if len(storage_accounts) == 0: + return [] + + tasks = { + asyncio.ensure_future( + self.get_and_set_activity_logs(storage_account) + ) for storage_account in storage_accounts + } + await asyncio.wait(tasks) + + return storage_accounts async def get_blob_containers(self, resource_group_name, storage_account_name): return await run_concurrently( lambda: self._client.blob_containers.list(resource_group_name, storage_account_name).value ) + + async def get_and_set_activity_logs(self, storage_account): + client = MonitorManagementClient(self._credentials, self._subscription_id) + + # time format used by Azure API: + time_format = "%Y-%m-%dT%H:%M:%S.%f" + # Azure API uses UTC time, we need to use the same to avoid bad requests: + utc_now = datetime.datetime.utcnow() + # activity logs are only archived for a period of 90 days max (requesting a timespan of more than that ends up + # with a bad request): + timespan = datetime.timedelta(90) + + logs_filter = " and ".join([ + "eventTimestamp ge {}".format((utc_now - timespan).strftime(time_format)), + "eventTimestamp le {}".format(utc_now.strftime(time_format)), + "resourceId eq {}".format(storage_account.id), + ]) + activity_logs = await run_concurrently( + lambda: list(client.activity_logs.list(filter=logs_filter, select="eventTimestamp, operationName")) + ) + setattr(storage_account, 'activity_logs', activity_logs) From 55866edaa86a6f7f8887ca437c1f59d68b6014e9 Mon Sep 17 00:00:00 2001 From: misg Date: Wed, 27 Mar 2019 00:10:06 -0400 Subject: [PATCH 597/667] Parse activity logs. --- .../azure/resources/storageaccounts/storageaccounts.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ScoutSuite/providers/azure/resources/storageaccounts/storageaccounts.py b/ScoutSuite/providers/azure/resources/storageaccounts/storageaccounts.py index f96f5f80d..7dfe72ad9 100644 --- a/ScoutSuite/providers/azure/resources/storageaccounts/storageaccounts.py +++ b/ScoutSuite/providers/azure/resources/storageaccounts/storageaccounts.py @@ -47,9 +47,18 @@ def _parse_storage_account(self, raw_storage_account): storage_account['name'] = raw_storage_account.name storage_account['https_traffic_enabled'] = raw_storage_account.enable_https_traffic_only storage_account['public_traffic_allowed'] = self._is_public_traffic_allowed(raw_storage_account) + storage_account['access_keys_last_rotation_date'] =\ + self._parse_access_keys_last_rotation_date(raw_storage_account.activity_logs) return storage_account['id'], storage_account def _is_public_traffic_allowed(self, storage_account): return storage_account.network_rule_set.default_action == "Allow" + def _parse_access_keys_last_rotation_date(self, activity_logs): + last_rotation_date = None + for log in activity_logs: + if log.operation_name.value == 'Microsoft.Storage/storageAccounts/regenerateKey/action': + if last_rotation_date is None or last_rotation_date < log.event_timestamp: + last_rotation_date = log.event_timestamp + return last_rotation_date From eafddf78f67719abf44b158bf310d32632d3895e Mon Sep 17 00:00:00 2001 From: misg Date: Wed, 27 Mar 2019 00:10:27 -0400 Subject: [PATCH 598/667] Update rule conditions. --- .../findings/storageaccount-access-keys-not-rotated.json | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ScoutSuite/providers/azure/rules/findings/storageaccount-access-keys-not-rotated.json b/ScoutSuite/providers/azure/rules/findings/storageaccount-access-keys-not-rotated.json index 218c7767d..5f33ce0c1 100644 --- a/ScoutSuite/providers/azure/rules/findings/storageaccount-access-keys-not-rotated.json +++ b/ScoutSuite/providers/azure/rules/findings/storageaccount-access-keys-not-rotated.json @@ -1,10 +1,11 @@ { "dashboard_name": "Storage Accounts", "description": "Access Keys Not Rotated", - "rationale": "The access keys of your storage accounts should be rotated at least every 90 days. See CIS 3.3.", + "rationale": "The access keys of your storage accounts should be rotated at least every _ARG_0_ days. See CIS 3.3.", "path": "storageaccounts.storage_accounts.id", - "conditions": [ "and", - [ "monitor.activity_logs.storage_accounts._GET_VALUE_AT_(storageaccounts.storage_accounts.id).access_keys_rotated", "false", "" ] + "conditions": [ "or", + ["storageaccounts.storage_accounts.id.access_keys_last_rotation_date", "equal", "None"], + [ "storageaccounts.storage_accounts.id.access_keys_last_rotation_date", "olderThan", ["_ARG_0_", "days"] ] ], "id_suffix": "access_keys_rotated" } \ No newline at end of file From 135bdab173a0d62634dc2a2e2c4df91a736b943d Mon Sep 17 00:00:00 2001 From: misg Date: Wed, 27 Mar 2019 00:10:41 -0400 Subject: [PATCH 599/667] Update HTML template. --- .../services.storageaccounts.storage_accounts.html | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/output/data/html/partials/azure/services.storageaccounts.storage_accounts.html b/ScoutSuite/output/data/html/partials/azure/services.storageaccounts.storage_accounts.html index 00892a6f2..7cd1bebb8 100644 --- a/ScoutSuite/output/data/html/partials/azure/services.storageaccounts.storage_accounts.html +++ b/ScoutSuite/output/data/html/partials/azure/services.storageaccounts.storage_accounts.html @@ -9,7 +9,15 @@

            Information

            Storage Account Name: {{name}}
            HTTPS required: {{https_traffic_enabled}}
            Public traffic: {{ convert_bool_to_enabled public_traffic_allowed }}
            -
            Access Key Rotated: {{ get_value_at 'services' 'monitor' 'activity_logs' 'storage_accounts' @key 'access_keys_rotated' }}
            +
            Last access keys rotation: + + {{#if access_keys_last_rotation_date }} + {{ access_keys_last_rotation_date }} + {{else}} + never + {{/if}} + +
            From 02d2ceca875a326463beda75ea5469d93419ec59 Mon Sep 17 00:00:00 2001 From: misg Date: Wed, 27 Mar 2019 00:11:07 -0400 Subject: [PATCH 600/667] Add default period for key rotations. --- ScoutSuite/providers/azure/rules/rulesets/default.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ScoutSuite/providers/azure/rules/rulesets/default.json b/ScoutSuite/providers/azure/rules/rulesets/default.json index c83ccff09..9b6b30223 100644 --- a/ScoutSuite/providers/azure/rules/rulesets/default.json +++ b/ScoutSuite/providers/azure/rules/rulesets/default.json @@ -56,6 +56,9 @@ ], "storageaccount-access-keys-not-rotated.json": [ { + "args": [ + "90" + ], "enabled": true, "level": "warning" } From 79eca5e4313a3d40fae01279dcd58cf500bffbcf Mon Sep 17 00:00:00 2001 From: misg Date: Wed, 27 Mar 2019 00:11:48 -0400 Subject: [PATCH 601/667] Remove old monitor service. --- .../output/data/inc-scoutsuite/scoutsuite.js | 2 -- .../providers/azure/configs/services.py | 2 -- ScoutSuite/providers/azure/facade/monitor.py | 21 -------------- .../azure/resources/monitor/activity_logs.py | 29 ------------------- ScoutSuite/providers/azure/utils.py | 3 -- ScoutSuite/utils.py | 1 - 6 files changed, 58 deletions(-) delete mode 100644 ScoutSuite/providers/azure/facade/monitor.py delete mode 100644 ScoutSuite/providers/azure/resources/monitor/activity_logs.py diff --git a/ScoutSuite/output/data/inc-scoutsuite/scoutsuite.js b/ScoutSuite/output/data/inc-scoutsuite/scoutsuite.js index 1ac1a0165..1ff3c4461 100644 --- a/ScoutSuite/output/data/inc-scoutsuite/scoutsuite.js +++ b/ScoutSuite/output/data/inc-scoutsuite/scoutsuite.js @@ -1035,8 +1035,6 @@ function make_title(title) { return 'Cloud Resource Manager'; } else if (title == 'storageaccounts') { return 'Storage Accounts'; - } else if (title == 'monitor') { - return 'Monitor'; } else if (title == 'sqldatabase') { return 'SQL Database'; } else if (title == 'securitycenter') { diff --git a/ScoutSuite/providers/azure/configs/services.py b/ScoutSuite/providers/azure/configs/services.py index fb78bc5e7..ecaed2f31 100644 --- a/ScoutSuite/providers/azure/configs/services.py +++ b/ScoutSuite/providers/azure/configs/services.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- from ScoutSuite.providers.base.configs.services import BaseServicesConfig -from ScoutSuite.providers.azure.resources.monitor.activity_logs import ActivityLogs from ScoutSuite.providers.azure.resources.securitycenter.security_center import SecurityCenter from ScoutSuite.providers.azure.resources.sqldatabase.servers import Servers from ScoutSuite.providers.azure.resources.storageaccounts.storageaccounts import StorageAccounts @@ -29,7 +28,6 @@ class AzureServicesConfig(BaseServicesConfig): def __init__(self, metadata=None, thread_config=4, **kwargs): - self.monitor = ActivityLogs() self.storageaccounts = StorageAccounts() self.securitycenter = SecurityCenter() self.sqldatabase = Servers() diff --git a/ScoutSuite/providers/azure/facade/monitor.py b/ScoutSuite/providers/azure/facade/monitor.py deleted file mode 100644 index c99db2344..000000000 --- a/ScoutSuite/providers/azure/facade/monitor.py +++ /dev/null @@ -1,21 +0,0 @@ -import datetime - -from azure.mgmt.monitor import MonitorManagementClient -from ScoutSuite.providers.utils import run_concurrently - - -class MonitorFacade: - def __init__(self, credentials, subscription_id): - self._client = MonitorManagementClient(credentials, subscription_id) - - async def get_activity_logs(self): - time_format = "%Y-%m-%dT%H:%M:%S.%f" - utc_now = datetime.datetime.utcnow() - end_time = utc_now.strftime(time_format) - timespan = datetime.timedelta(90) # 90 days of timespan - start_time = (utc_now - timespan).strftime(time_format) - - return await run_concurrently( - lambda: self._client.activity_logs.list( - filter="eventTimestamp ge {} and eventTimestamp le {}".format(start_time, end_time)) - ) diff --git a/ScoutSuite/providers/azure/resources/monitor/activity_logs.py b/ScoutSuite/providers/azure/resources/monitor/activity_logs.py deleted file mode 100644 index 4ad2ad444..000000000 --- a/ScoutSuite/providers/azure/resources/monitor/activity_logs.py +++ /dev/null @@ -1,29 +0,0 @@ -from ScoutSuite.providers.base.configs.resources import Resources -from ScoutSuite.providers.azure.facade.monitor import MonitorFacade -from ScoutSuite.providers.utils import get_non_provider_id - - -class ActivityLogs(Resources): - async def fetch_all(self, credentials, **kwargs): - # TODO: build that facade somewhere else: - facade = MonitorFacade(credentials.credentials, credentials.subscription_id) - - self['activity_logs'] = {} - self['activity_logs']['storage_accounts'] = {} - - for raw_log in await facade.get_activity_logs(): - self._parse_activity_log(raw_log) - - def _parse_activity_log(self, raw_log): - if raw_log.resource_type.value == 'Microsoft.Storage/storageAccounts': - self._parse_storage_account_log(raw_log) - - def _parse_storage_account_log(self, raw_log): - storage_account_id = get_non_provider_id(raw_log.resource_id.lower()) - - if storage_account_id not in self['activity_logs']['storage_accounts']: - self['activity_logs']['storage_accounts'][storage_account_id] =\ - {'access_keys_rotated': False} - - if raw_log.operation_name.value == 'Microsoft.Storage/storageAccounts/regenerateKey/action': - self['activity_logs']['storage_accounts'][storage_account_id]['access_keys_rotated'] = True diff --git a/ScoutSuite/providers/azure/utils.py b/ScoutSuite/providers/azure/utils.py index 3eadb44e0..13a191316 100644 --- a/ScoutSuite/providers/azure/utils.py +++ b/ScoutSuite/providers/azure/utils.py @@ -5,7 +5,6 @@ from ScoutSuite.core.console import print_exception from azure.mgmt.storage import StorageManagementClient -from azure.mgmt.monitor import MonitorManagementClient from azure.mgmt.sql import SqlManagementClient from azure.mgmt.security import SecurityCenter @@ -19,8 +18,6 @@ def azure_connect_service(service, credentials, region_name=None): try: if service == 'storageaccounts': return StorageManagementClient(credentials.credentials, credentials.subscription_id) - elif service == 'monitor': - return MonitorManagementClient(credentials.credentials, credentials.subscription_id) elif service == 'sqldatabase': return SqlManagementClient(credentials.credentials, credentials.subscription_id) elif service == 'keyvault': diff --git a/ScoutSuite/utils.py b/ScoutSuite/utils.py index fc0198ae2..acdd23277 100644 --- a/ScoutSuite/utils.py +++ b/ScoutSuite/utils.py @@ -22,7 +22,6 @@ 'kubernetesengine': 'Kubernetes Engine', 'cloudresourcemanager': 'Cloud Resource Manager', - 'monitor': 'Monitor', 'storageaccounts': 'Storage Accounts', 'sqldatabase': 'SQL Database', 'securitycenter': 'Security Center', From 3f128b2d4198003212d3694736cc7805970a6d64 Mon Sep 17 00:00:00 2001 From: misg Date: Wed, 27 Mar 2019 00:49:49 -0400 Subject: [PATCH 602/667] Pull azure resources (really) concurrently. --- ScoutSuite/providers/azure/facade/keyvault.py | 4 +++- ScoutSuite/providers/azure/facade/network.py | 8 ++++++-- .../providers/azure/facade/securitycenter.py | 12 ++++++++--- .../providers/azure/facade/sqldatabase.py | 20 ++++++++++--------- .../providers/azure/facade/storageaccounts.py | 6 ++++-- 5 files changed, 33 insertions(+), 17 deletions(-) diff --git a/ScoutSuite/providers/azure/facade/keyvault.py b/ScoutSuite/providers/azure/facade/keyvault.py index da55faabf..02de01495 100644 --- a/ScoutSuite/providers/azure/facade/keyvault.py +++ b/ScoutSuite/providers/azure/facade/keyvault.py @@ -7,4 +7,6 @@ def __init__(self, credentials, subscription_id): self._client = KeyVaultManagementClient(credentials, subscription_id) async def get_key_vaults(self): - return await run_concurrently(self._client.vaults.list_by_subscription) + return await run_concurrently( + lambda: list(self._client.vaults.list_by_subscription()) + ) diff --git a/ScoutSuite/providers/azure/facade/network.py b/ScoutSuite/providers/azure/facade/network.py index 1b7ede9f8..2d51ea873 100644 --- a/ScoutSuite/providers/azure/facade/network.py +++ b/ScoutSuite/providers/azure/facade/network.py @@ -7,7 +7,11 @@ def __init__(self, credentials, subscription_id): self._client = NetworkManagementClient(credentials, subscription_id) async def get_network_watchers(self): - return await run_concurrently(self._client.network_watchers.list_all) + return await run_concurrently( + lambda: list(self._client.network_watchers.list_all()) + ) async def get_network_security_groups(self): - return await run_concurrently(self._client.network_security_groups.list_all) + return await run_concurrently( + lambda: list(self._client.network_security_groups.list_all()) + ) diff --git a/ScoutSuite/providers/azure/facade/securitycenter.py b/ScoutSuite/providers/azure/facade/securitycenter.py index a1625f7ed..c88e265e2 100644 --- a/ScoutSuite/providers/azure/facade/securitycenter.py +++ b/ScoutSuite/providers/azure/facade/securitycenter.py @@ -7,10 +7,16 @@ def __init__(self, credentials, subscription_id): self._client = SecurityCenter(credentials, subscription_id, '') async def get_pricings(self): - return await run_concurrently(self._client.pricings.list) + return await run_concurrently( + lambda: list(self._client.pricings.list()) + ) async def get_security_contacts(self): - return await run_concurrently(self._client.security_contacts.list) + return await run_concurrently( + lambda: list(self._client.security_contacts.list()) + ) async def get_auto_provisioning_settings(self): - return await run_concurrently(self._client.auto_provisioning_settings.list) + return await run_concurrently( + lambda: list(self._client.auto_provisioning_settings.list()) + ) diff --git a/ScoutSuite/providers/azure/facade/sqldatabase.py b/ScoutSuite/providers/azure/facade/sqldatabase.py index 937585108..b98686710 100644 --- a/ScoutSuite/providers/azure/facade/sqldatabase.py +++ b/ScoutSuite/providers/azure/facade/sqldatabase.py @@ -8,19 +8,19 @@ def __init__(self, credentials, subscription_id): async def get_database_blob_auditing_policies(self, resource_group_name, server_name, database_name): return await run_concurrently( - lambda: self._client.database_blob_auditing_policies.get( - resource_group_name, server_name, database_name) + lambda: list(self._client.database_blob_auditing_policies.get( + resource_group_name, server_name, database_name)) ) async def get_database_threat_detection_policies(self, resource_group_name, server_name, database_name): return await run_concurrently( - lambda: self._client.database_threat_detection_policies.get( - resource_group_name, server_name, database_name) + lambda: list(self._client.database_threat_detection_policies.get( + resource_group_name, server_name, database_name)) ) async def get_databases(self, resource_group_name, server_name): return await run_concurrently( - lambda: self._client.databases.list_by_server(resource_group_name, server_name) + lambda: list(self._client.databases.list_by_server(resource_group_name, server_name)) ) async def get_database_replication_links(self, resource_group_name, server_name, database_name): @@ -31,7 +31,7 @@ async def get_database_replication_links(self, resource_group_name, server_name, async def get_server_azure_ad_administrators(self, resource_group_name, server_name): return await run_concurrently( - lambda: self._client.server_azure_ad_administrators.get(resource_group_name, server_name) + lambda: list(self._client.server_azure_ad_administrators.get(resource_group_name, server_name)) ) async def get_server_blob_auditing_policies(self, resource_group_name, server_name): @@ -45,10 +45,12 @@ async def get_server_security_alert_policies(self, resource_group_name, server_n ) async def get_servers(self): - return await run_concurrently(self._client.servers.list) + return await run_concurrently( + lambda: list(self._client.servers.list()) + ) async def get_database_transparent_data_encryptions(self, resource_group_name, server_name, database_name): return await run_concurrently( - lambda: self._client.transparent_data_encryptions.get( - resource_group_name, server_name, database_name) + lambda: list(self._client.transparent_data_encryptions.get( + resource_group_name, server_name, database_name)) ) diff --git a/ScoutSuite/providers/azure/facade/storageaccounts.py b/ScoutSuite/providers/azure/facade/storageaccounts.py index ee1fc19c1..aa4b3ea18 100644 --- a/ScoutSuite/providers/azure/facade/storageaccounts.py +++ b/ScoutSuite/providers/azure/facade/storageaccounts.py @@ -7,9 +7,11 @@ def __init__(self, credentials, subscription_id): self._client = StorageManagementClient(credentials, subscription_id) async def get_storage_accounts(self): - return await run_concurrently(self._client.storage_accounts.list) + return await run_concurrently( + lambda: list(self._client.storage_accounts.list()) + ) async def get_blob_containers(self, resource_group_name, storage_account_name): return await run_concurrently( - lambda: self._client.blob_containers.list(resource_group_name, storage_account_name).value + lambda: list(self._client.blob_containers.list(resource_group_name, storage_account_name).value) ) From a7742f50fc84e50762382be35f96a776b1a8b240 Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Wed, 27 Mar 2019 11:57:32 +0100 Subject: [PATCH 603/667] Small changes and addition of contact email. --- CODE_OF_CONDUCT.md | 2 +- CONTRIBUTING.md | 5 +++-- README.md | 2 ++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 91ad72f7f..6ef56aaba 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -32,7 +32,7 @@ This Code of Conduct applies both within project spaces and in public spaces whe ## Enforcement -Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at . All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project’s leadership. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f3e915df3..af322d720 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,5 @@ # Contributing + ## Table of Contents * [Introduction](#introduction) * [Getting started](#getting-started) @@ -32,7 +33,7 @@ So you want to contribute some code, that's great! This project follows the [Git ## How to report a bug - When filing an issue, make sure to answer these five questions: +When filing an issue, make sure to answer these five questions: 1. What version of Python are you using? 2. What operating system and processor architecture are you using? @@ -46,4 +47,4 @@ If you find yourself wishing for a feature that doesn't exist in Scout Suite, yo ## Code review process -The pull request are reviewed on a regular basis by the core team. We require a minimum of two reviewers before it is merged. +Pull requests are regularly reviewed by the core team. We require a minimum of two reviewers before allowing to merge. diff --git a/README.md b/README.md index f2a58fb36..e9378b630 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,8 @@ Scout Suite is stable and actively maintained, but a number of features and inte with us as we find time to work on, and improve, the tool. Feel free to report a bug with details (please provide console output using the `--debug` argument), request a new feature, or send a pull request. +The project team can be contacted by writing to . + **Note:** The latest (and final) version of Scout2 can be found in and From 410f8b454db8ec9033c7b02b204390ac9f817563 Mon Sep 17 00:00:00 2001 From: Xavier Garceau-Aranda Date: Wed, 27 Mar 2019 11:58:55 +0100 Subject: [PATCH 604/667] Change working --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e9378b630..03762e267 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Scout Suite is stable and actively maintained, but a number of features and inte with us as we find time to work on, and improve, the tool. Feel free to report a bug with details (please provide console output using the `--debug` argument), request a new feature, or send a pull request. -The project team can be contacted by writing to . +The project team can be contacted at . **Note:** From 54689f284f59daafe479f314b6c8e653411f7c32 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 27 Mar 2019 10:10:19 -0400 Subject: [PATCH 605/667] Fix comments Co-Authored-By: misg --- ScoutSuite/providers/azure/facade/storageaccounts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/azure/facade/storageaccounts.py b/ScoutSuite/providers/azure/facade/storageaccounts.py index 096cc4842..eac8721bb 100644 --- a/ScoutSuite/providers/azure/facade/storageaccounts.py +++ b/ScoutSuite/providers/azure/facade/storageaccounts.py @@ -37,11 +37,11 @@ async def get_blob_containers(self, resource_group_name, storage_account_name): async def get_and_set_activity_logs(self, storage_account): client = MonitorManagementClient(self._credentials, self._subscription_id) - # time format used by Azure API: + # Time format used by Azure API: time_format = "%Y-%m-%dT%H:%M:%S.%f" # Azure API uses UTC time, we need to use the same to avoid bad requests: utc_now = datetime.datetime.utcnow() - # activity logs are only archived for a period of 90 days max (requesting a timespan of more than that ends up + # Activity logs are only archived for a period of 90 days max (requesting a timespan of more than that ends up # with a bad request): timespan = datetime.timedelta(90) From bbc968b7ad802b773724dfa79784e7009de75322 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 27 Mar 2019 11:28:22 -0400 Subject: [PATCH 606/667] Implemented base authentication strategy --- .../providers/base/authentication_strategy.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 ScoutSuite/providers/base/authentication_strategy.py diff --git a/ScoutSuite/providers/base/authentication_strategy.py b/ScoutSuite/providers/base/authentication_strategy.py new file mode 100644 index 000000000..82197ce13 --- /dev/null +++ b/ScoutSuite/providers/base/authentication_strategy.py @@ -0,0 +1,17 @@ +from abc import ABCMeta, abstractmethod + + +class AuthenticationStrategy(metaclass=ABCMeta): + """ + This class represents an authentication strategy. + """ + + @abstractmethod + def authenticate(self, **kwargs): + """ + Given parameters, this authenticates the user to a provider and returns a credentials object. + """ + raise NotImplementedError() + +class AuthenticationException(Exception): + pass \ No newline at end of file From 00f499364a7ddeaf57e4fc24e8cb019a865c6720 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 27 Mar 2019 11:29:15 -0400 Subject: [PATCH 607/667] Implemented AWS authentication strategy --- .../providers/gcp/authentication_strategy.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 ScoutSuite/providers/gcp/authentication_strategy.py diff --git a/ScoutSuite/providers/gcp/authentication_strategy.py b/ScoutSuite/providers/gcp/authentication_strategy.py new file mode 100644 index 000000000..0871019d1 --- /dev/null +++ b/ScoutSuite/providers/gcp/authentication_strategy.py @@ -0,0 +1,32 @@ +import google +import os +import warnings + +from ScoutSuite.core.console import print_error +from ScoutSuite.providers.base.authentication_strategy import AuthenticationStrategy, AuthenticationException + +class GCPAuthenticationStrategy(AuthenticationStrategy): + + def authenticate(self, user_account=None, service_account=None, **kargs): + """ + Implements authentication for the GCP provider + Refer to https://google-auth.readthedocs.io/en/stable/reference/google.auth.html. + """ + + if user_account: + # disable GCP warning about using User Accounts + warnings.filterwarnings("ignore", "Your application has authenticated using end user credentials") + pass # Nothing more to do + elif service_account: + client_secrets_path = os.path.abspath(service_account) + os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = client_secrets_path + else: + print_error('Failed to authenticate to GCP - no supported account type') + raise AuthenticationException() + + credentials, _ = google.auth.default() + if not credentials: + raise AuthenticationException() + + credentials.is_service_account = service_account is not None + return credentials \ No newline at end of file From 8a78e243d2165cb5e014146095b7d16bd445012d Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 27 Mar 2019 11:29:27 -0400 Subject: [PATCH 608/667] Implemented AWS authentication strategy --- .../providers/aws/authentication_strategy.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 ScoutSuite/providers/aws/authentication_strategy.py diff --git a/ScoutSuite/providers/aws/authentication_strategy.py b/ScoutSuite/providers/aws/authentication_strategy.py new file mode 100644 index 000000000..9faf911f0 --- /dev/null +++ b/ScoutSuite/providers/aws/authentication_strategy.py @@ -0,0 +1,17 @@ +import boto3 + +from ScoutSuite.providers.base.authentication_strategy import AuthenticationStrategy, AuthenticationException + +class AWSAuthenticationStrategy(AuthenticationStrategy): + """ + Implements authentication for the AWS provider + """ + + def authenticate(self, profile=None, **kwargs): + session = boto3.Session(profile_name=profile) + credentials = session.get_credentials().__dict__ + + if credentials.get('access_key') is None: + raise AuthenticationException() + + return credentials \ No newline at end of file From af0c03a5be044fc8a4e3602f0f542eee738be5da Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 27 Mar 2019 11:29:36 -0400 Subject: [PATCH 609/667] Implemented authentication strategy factory --- .../providers/base/authentication_strategy_factory.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 ScoutSuite/providers/base/authentication_strategy_factory.py diff --git a/ScoutSuite/providers/base/authentication_strategy_factory.py b/ScoutSuite/providers/base/authentication_strategy_factory.py new file mode 100644 index 000000000..d60548923 --- /dev/null +++ b/ScoutSuite/providers/base/authentication_strategy_factory.py @@ -0,0 +1,11 @@ +from ScoutSuite.providers.aws.authentication_strategy import AWSAuthenticationStrategy +from ScoutSuite.providers.gcp.authentication_strategy import GCPAuthenticationStrategy + +_strategies = { + 'aws': AWSAuthenticationStrategy, + 'gcp': GCPAuthenticationStrategy +} + + +def get_authentication_strategy(provider: str): + return _strategies[provider]() From 5a32781e0f32411a58faebab42d4cb78bc24588c Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 27 Mar 2019 11:30:30 -0400 Subject: [PATCH 610/667] Started authentication strategy integration --- ScoutSuite/__main__.py | 74 +++++++--- ScoutSuite/providers/aws/provider.py | 194 +++++++++++++++++--------- ScoutSuite/providers/base/provider.py | 4 +- ScoutSuite/providers/gcp/provider.py | 139 ++++++++---------- 4 files changed, 240 insertions(+), 171 deletions(-) diff --git a/ScoutSuite/__main__.py b/ScoutSuite/__main__.py index ad0b8ab4a..2f1457279 100644 --- a/ScoutSuite/__main__.py +++ b/ScoutSuite/__main__.py @@ -13,6 +13,7 @@ from ScoutSuite.core.ruleset import Ruleset from ScoutSuite.output.html import Scout2Report from ScoutSuite.providers import get_provider +from ScoutSuite.providers.base.authentication_strategy_factory import get_authentication_strategy # noinspection PyBroadException @@ -33,6 +34,27 @@ async def main(args=None): print_info('Launching Scout') + auth_strategy = get_authentication_strategy(args.get('provider')) + if not args.get('fetch_local'): + credentials = auth_strategy.authenticate(profile=args.get('profile'), + user_account=args.get('user_account'), + service_account=args.get('service_account'), + cli=args.get('cli'), + msi=args.get('msi'), + service_principal=args.get('service_principal'), + file_auth=args.get('file_auth'), + tenant_id=args.get('tenant_id'), + subscription_id=args.get('subscription_id'), + client_id=args.get('client_id'), + client_secret=args.get('client_secret'), + username=args.get('username'), + password=args.get('password') + ) + + if not credentials: + return 401 + + # Create a cloud provider object cloud_provider = get_provider(provider=args.get('provider'), profile=args.get('profile'), @@ -44,37 +66,35 @@ async def main(args=None): timestamp=args.get('timestamp'), services=args.get('services'), skipped_services=args.get('skipped_services'), - thread_config=args.get('thread_config')) + thread_config=args.get('thread_config'), + credentials=credentials) report_file_name = generate_report_name(cloud_provider.provider_code, args) # TODO: move this to after authentication, so that the report can be more specific to what's being scanned. # For example if scanning with a GCP service account, the SA email can only be known after authenticating... # Create a new report - report = Scout2Report(args.get('provider'), report_file_name, args.get( - 'report_dir'), args.get('timestamp')) + report = Scout2Report(args.get('provider'), report_file_name, args.get('report_dir'), args.get('timestamp')) # Complete run, including pulling data from provider if not args.get('fetch_local'): - print_info('Authenticating to {}'.format(cloud_provider.provider_name)) - # Authenticate to the cloud provider - authenticated = cloud_provider.authenticate(profile=args.get('profile'), - user_account=args.get('user_account'), - service_account=args.get('service_account'), - cli=args.get('cli'), - msi=args.get('msi'), - service_principal=args.get('service_principal'), - file_auth=args.get('file_auth'), - tenant_id=args.get('tenant_id'), - subscription_id=args.get('subscription_id'), - client_id=args.get('client_id'), - client_secret=args.get('client_secret'), - username=args.get('username'), - password=args.get('password')) - - if not authenticated: - print_error("Authentication failure") - return 401 + # # Authenticate to the cloud provider + # authenticated = cloud_provider.authenticate(profile=args.get('profile'), + # user_account=args.get('user_account'), + # service_account=args.get('service_account'), + # cli=args.get('cli'), + # msi=args.get('msi'), + # service_principal=args.get('service_principal'), + # file_auth=args.get('file_auth'), + # tenant_id=args.get('tenant_id'), + # subscription_id=args.get('subscription_id'), + # client_id=args.get('client_id'), + # client_secret=args.get('client_secret'), + # username=args.get('username'), + # password=args.get('password')) + + # if not authenticated: + # return 401 # Fetch data from provider APIs try: @@ -135,6 +155,16 @@ async def main(args=None): exceptions = {} else: exceptions = {} + # Handle exceptions + try: + exceptions = RuleExceptions( + args.get('profile'), args.get('exceptions')[0]) + exceptions.process(cloud_provider) + exceptions = exceptions.exceptions + except Exception as e: + print_debug( + 'Warning, failed to load exceptions. The file may not exist or may have an invalid format.') + exceptions = {} # Finalize cloud_provider.postprocessing(report.current_time, finding_rules) diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index 403899f6f..a76913be3 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -18,12 +18,12 @@ class AWSProvider(BaseProvider): Implements provider for AWS """ - def __init__(self, profile='default', report_dir=None, timestamp=None, services=None, skipped_services=None, - thread_config=4, **kwargs): + def __init__(self, profile='default', report_dir=None, timestamp=None, services=None, skipped_services=None, thread_config=4, **kwargs): services = [] if services is None else services skipped_services = [] if skipped_services is None else skipped_services - self.metadata_path = '%s/metadata.json' % os.path.split(os.path.abspath(__file__))[0] + self.metadata_path = '%s/metadata.json' % os.path.split( + os.path.abspath(__file__))[0] self.sg_map = {} self.subnet_map = {} @@ -35,19 +35,19 @@ def __init__(self, profile='default', report_dir=None, timestamp=None, services= self.provider_code = 'aws' self.provider_name = 'Amazon Web Services' - super(AWSProvider, self).__init__(report_dir, timestamp, services, skipped_services, thread_config) + self.credentials = kwargs['credentials'] + self.aws_account_id = get_aws_account_id(self.credentials) + + super(AWSProvider, self).__init__(report_dir, timestamp, + services, skipped_services, thread_config) + def authenticate(self, profile=None, **kwargs): """ Implement authentication for the AWS provider :return: """ - - session = boto3.Session(profile_name=profile) - self.credentials = session.get_credentials().__dict__ - self.aws_account_id = get_aws_account_id(self.credentials) - - return self.credentials.get('access_key') is not None + return True def preprocessing(self, ip_ranges=None, ip_ranges_name_key=None): """ @@ -73,6 +73,8 @@ def preprocessing(self, ip_ranges=None, ip_ranges_name_key=None): self._check_ec2_zone_distribution() self._add_security_group_name_to_ec2_grants() self._add_last_snapshot_date_to_ec2_volumes() + + if 'ec2' in self.service_list and 'iam' in self.service_list: self._match_instances_and_roles() if 'cloudtrail' in self.service_list: @@ -83,7 +85,7 @@ def preprocessing(self, ip_ranges=None, ip_ranges_name_key=None): if 's3' in self.service_list and 'iam' in self.service_list: self._match_iam_policies_and_buckets() - + self._add_cidr_display_name(ip_ranges, ip_ranges_name_key) super(AWSProvider, self).preprocessing() @@ -99,9 +101,11 @@ def postprocessing(self, current_time, ruleset): def _add_cidr_display_name(self, ip_ranges, ip_ranges_name_key): if len(ip_ranges): - callback_args = {'ip_ranges': ip_ranges, 'ip_ranges_name_key': ip_ranges_name_key} + callback_args = {'ip_ranges': ip_ranges, + 'ip_ranges_name_key': ip_ranges_name_key} self._go_to_and_do(self.services['ec2'], - ['regions', 'vpcs', 'security_groups', 'rules', 'protocols', 'ports'], + ['regions', 'vpcs', 'security_groups', + 'rules', 'protocols', 'ports'], ['services', 'ec2'], put_cidr_name, callback_args) @@ -111,7 +115,8 @@ def _add_security_group_name_to_ec2_grants(self): Github issue #24: display the security group names in the list of grants (added here to have ligher JS code) """ self._go_to_and_do(self.services['ec2'], - ['regions', 'vpcs', 'security_groups', 'rules', 'protocols', 'ports', 'security_groups'], + ['regions', 'vpcs', 'security_groups', 'rules', + 'protocols', 'ports', 'security_groups'], [], self.add_security_group_name_to_ec2_grants_callback, {'AWSAccountId': self.aws_account_id}) @@ -137,7 +142,7 @@ def check_security_group_rules(lb, index, traffic_type): for i in range(0, len(elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'])): for sg in ec2_config['regions'][region]['vpcs'][vpc]['security_groups']: group_id = elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][i][ - 'GroupId'] + 'GroupId'] if 'GroupId' in elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][ i] and group_id == sg: elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][i] = \ @@ -145,27 +150,33 @@ def check_security_group_rules(lb, index, traffic_type): elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb]['security_groups'][i][ 'GroupId'] = group_id - check_security_group_rules(elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb], i, 'ingress') - check_security_group_rules(elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb], i, 'egress') + check_security_group_rules( + elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb], i, 'ingress') + check_security_group_rules( + elbv2_config['regions'][region]['vpcs'][vpc]['lbs'][lb], i, 'egress') def _check_ec2_zone_distribution(self): regions = self.services['ec2']['regions'].values() - self.services['ec2']['number_of_regions_with_instances'] = sum(r['instances_count'] > 0 for r in regions) + self.services['ec2']['number_of_regions_with_instances'] = sum( + r['instances_count'] > 0 for r in regions) def _add_last_snapshot_date_to_ec2_volumes(self): for region in self.services['ec2']['regions'].values(): for volumeId, volume in region.get('volumes').items(): completed_snapshots = [s for s in region['snapshots'].values() if s['VolumeId'] == volumeId and s['State'] == 'completed'] - sorted_snapshots = sorted(completed_snapshots, key=lambda s: s['StartTime'], reverse=True) - volume['LastSnapshotDate'] = sorted_snapshots[0]['StartTime'] if len(sorted_snapshots) > 0 else None + sorted_snapshots = sorted( + completed_snapshots, key=lambda s: s['StartTime'], reverse=True) + volume['LastSnapshotDate'] = sorted_snapshots[0]['StartTime'] if len( + sorted_snapshots) > 0 else None def add_security_group_name_to_ec2_grants_callback(self, current_config, path, current_path, ec2_grant, callback_args): sg_id = ec2_grant['GroupId'] if sg_id in current_path: target = current_path[:(current_path.index(sg_id) + 1)] - ec2_grant['GroupName'] = get_value_at(self.services['ec2'], target, 'name') + ec2_grant['GroupName'] = get_value_at( + self.services['ec2'], target, 'name') elif ec2_grant['UserId'] == callback_args['AWSAccountId']: if 'VpcId' in ec2_grant: target = current_path[:(current_path.index('vpcs') + 1)] @@ -173,9 +184,11 @@ def add_security_group_name_to_ec2_grants_callback(self, current_config, path, c target.append('security_groups') target.append(sg_id) else: - target = current_path[:(current_path.index('security_groups') + 1)] + target = current_path[:( + current_path.index('security_groups') + 1)] target.append(sg_id) - ec2_grant['GroupName'] = get_value_at(self.services['ec2'], target, 'name') + ec2_grant['GroupName'] = get_value_at( + self.services['ec2'], target, 'name') @staticmethod def _process_cloudtrail_trails(cloudtrail_config): @@ -195,16 +208,22 @@ def _process_cloudtrail_trails(cloudtrail_config): data_logging_trails_count += 1 cloudtrail_config['data_logging_trails_count'] = data_logging_trails_count - cloudtrail_config['IncludeGlobalServiceEvents'] = len(global_events_logging) > 0 - cloudtrail_config['DuplicatedGlobalServiceEvents'] = len(global_events_logging) > 1 + cloudtrail_config['IncludeGlobalServiceEvents'] = len( + global_events_logging) > 0 + cloudtrail_config['DuplicatedGlobalServiceEvents'] = len( + global_events_logging) > 1 def process_network_acls_callback(self, current_config, path, current_path, privateip_id, callback_args): # Check if the network ACL allows all traffic from all IP addresses - self._process_network_acls_check_for_allow_all(current_config, 'ingress') - self._process_network_acls_check_for_allow_all(current_config, 'egress') + self._process_network_acls_check_for_allow_all( + current_config, 'ingress') + self._process_network_acls_check_for_allow_all( + current_config, 'egress') # Check if the network ACL only has the default rules - self._process_network_acls_check_for_aws_default(current_config, 'ingress') - self._process_network_acls_check_for_aws_default(current_config, 'egress') + self._process_network_acls_check_for_aws_default( + current_config, 'ingress') + self._process_network_acls_check_for_aws_default( + current_config, 'egress') @staticmethod def _process_network_acls_check_for_allow_all(network_acl, direction): @@ -235,7 +254,8 @@ def list_ec2_network_attack_surface_callback(self, current_config, path, current public_ip = current_config['Association']['PublicIp'] self._security_group_to_attack_surface(self.services['ec2']['external_attack_surface'], public_ip, current_path, - [g['GroupId'] for g in current_config['Groups']], + [g['GroupId'] + for g in current_config['Groups']], []) # IPv6 if 'Ipv6Addresses' in current_config and len(current_config['Ipv6Addresses']) > 0: @@ -321,7 +341,8 @@ def _update_bucket_permissions(self, s3_info, iam_info, action, iam_entity, allo for bucket_name in s3_info['buckets']: allowed_buckets.append(bucket_name) if policy_type == 'InlinePolicies': - policy = iam_info[iam_entity.title()][allowed_iam_entity]['Policies'][policy_name]['PolicyDocument'] + policy = iam_info[iam_entity.title( + )][allowed_iam_entity]['Policies'][policy_name]['PolicyDocument'] elif policy_type == 'ManagedPolicies': policy = iam_info['ManagedPolicies'][policy_name]['PolicyDocument'] else: @@ -342,7 +363,8 @@ def _update_bucket_permissions(self, s3_info, iam_info, action, iam_entity, allo full_path][ policy_type][policy_name] for bucket_name in allowed_buckets: - self._update_iam_permissions(s3_info, bucket_name, iam_entity, allowed_iam_entity, policy_info) + self._update_iam_permissions( + s3_info, bucket_name, iam_entity, allowed_iam_entity, policy_info) def _update_iam_permissions(self, s3_info, bucket_name, iam_entity, allowed_iam_entity, policy_info): if bucket_name != '*' and bucket_name in s3_info['buckets']: @@ -354,14 +376,19 @@ def _update_iam_permissions(self, s3_info, bucket_name, iam_entity, allowed_iam_ bucket[iam_entity + '_count'] = bucket[iam_entity + '_count'] + 1 if 'inline_policies' in policy_info: - manage_dictionary(bucket[iam_entity][allowed_iam_entity], 'inline_policies', {}) - bucket[iam_entity][allowed_iam_entity]['inline_policies'].update(policy_info['inline_policies']) + manage_dictionary( + bucket[iam_entity][allowed_iam_entity], 'inline_policies', {}) + bucket[iam_entity][allowed_iam_entity]['inline_policies'].update( + policy_info['inline_policies']) if 'policies' in policy_info: - manage_dictionary(bucket[iam_entity][allowed_iam_entity], 'policies', {}) - bucket[iam_entity][allowed_iam_entity]['policies'].update(policy_info['policies']) + manage_dictionary(bucket[iam_entity] + [allowed_iam_entity], 'policies', {}) + bucket[iam_entity][allowed_iam_entity]['policies'].update( + policy_info['policies']) elif bucket_name == '*': for bucket in s3_info['buckets']: - self._update_iam_permissions(s3_info, bucket, iam_entity, allowed_iam_entity, policy_info) + self._update_iam_permissions( + s3_info, bucket, iam_entity, allowed_iam_entity, policy_info) pass else: # Could be an error or cross-account access, ignore... @@ -369,7 +396,8 @@ def _update_iam_permissions(self, s3_info, bucket_name, iam_entity, allowed_iam_ def match_network_acls_and_subnets_callback(self, current_config, path, current_path, acl_id, callback_args): for association in current_config['Associations']: - subnet_path = current_path[:-1] + ['subnets', association['SubnetId']] + subnet_path = current_path[:-1] + \ + ['subnets', association['SubnetId']] subnet = get_object_at(self, subnet_path) subnet['network_acl'] = acl_id @@ -377,7 +405,8 @@ def match_instances_and_subnets_callback(self, current_config, path, current_pat subnet_id = current_config['SubnetId'] if subnet_id: vpc = self.subnet_map[subnet_id] - subnet = self.services['vpc']['regions'][vpc['region']]['vpcs'][vpc['vpc_id']]['subnets'][subnet_id] + subnet = self.services['vpc']['regions'][vpc['region'] + ]['vpcs'][vpc['vpc_id']]['subnets'][subnet_id] manage_dictionary(subnet, 'instances', []) if instance_id not in subnet['instances']: subnet['instances'].append(instance_id) @@ -394,7 +423,8 @@ def _match_instances_and_roles(self): instance_profile = ec2_config['regions'][r]['vpcs'][v]['instances'][i]['IamInstanceProfile'] instance_profile_id = instance_profile['Id'] if instance_profile else None if instance_profile_id: - manage_dictionary(role_instances, instance_profile_id, []) + manage_dictionary( + role_instances, instance_profile_id, []) role_instances[instance_profile_id].append(i) for role_id in iam_config['roles']: iam_config['roles'][role_id]['instances_count'] = 0 @@ -402,7 +432,8 @@ def _match_instances_and_roles(self): if instance_profile_id in role_instances: iam_config['roles'][role_id]['instance_profiles'][instance_profile_id]['instances'] = \ role_instances[instance_profile_id] - iam_config['roles'][role_id]['instances_count'] += len(role_instances[instance_profile_id]) + iam_config['roles'][role_id]['instances_count'] += len( + role_instances[instance_profile_id]) def match_roles_and_cloudformation_stacks_callback(self, current_config, path, current_path, stack_id, callback_args): @@ -415,7 +446,8 @@ def match_roles_and_vpc_flowlogs_callback(self, current_config, path, current_pa if 'DeliverLogsPermissionArn' not in current_config: return delivery_role_arn = current_config.pop('DeliverLogsPermissionArn') - current_config['delivery_role'] = self._get_role_info('arn', delivery_role_arn) + current_config['delivery_role'] = self._get_role_info( + 'arn', delivery_role_arn) def _get_role_info(self, attribute_name, attribute_value): iam_role_info = {'name': None, 'id': None} @@ -429,26 +461,31 @@ def _get_role_info(self, attribute_name, attribute_value): def match_security_groups_and_resources_callback(self, current_config, path, current_path, resource_id, callback_args): service = current_path[1] - original_resource_path = combine_paths(copy.deepcopy(current_path), [resource_id]) + original_resource_path = combine_paths( + copy.deepcopy(current_path), [resource_id]) resource = get_object_at(self, original_resource_path) if 'resource_id_path' not in callback_args: resource_type = current_path[-1] resource_path = copy.deepcopy(current_path) resource_path.append(resource_id) else: - resource_path = combine_paths(copy.deepcopy(current_path), callback_args['resource_id_path']) + resource_path = combine_paths(copy.deepcopy( + current_path), callback_args['resource_id_path']) resource_id = resource_path[-1] resource_type = resource_path[-2] if 'status_path' in callback_args: - status_path = combine_paths(copy.deepcopy(original_resource_path), callback_args['status_path']) - resource_status = get_object_at(self, status_path).replace('.', '_') + status_path = combine_paths(copy.deepcopy( + original_resource_path), callback_args['status_path']) + resource_status = get_object_at( + self, status_path).replace('.', '_') else: resource_status = None unknown_vpc_id = True if current_path[4] != 'vpcs' else False # Issue 89 & 91 : can instances have no security group? try: try: - sg_attribute = get_object_at(resource, callback_args['sg_list_attribute_name']) + sg_attribute = get_object_at( + resource, callback_args['sg_list_attribute_name']) except Exception as e: return if type(sg_attribute) != list: @@ -462,7 +499,8 @@ def match_security_groups_and_resources_callback(self, current_config, path, cur vpc_id = self.sg_map[sg_id]['vpc_id'] sg_base_path = copy.deepcopy(current_path[0:4]) sg_base_path[1] = 'ec2' - sg_base_path = sg_base_path + ['vpcs', vpc_id, 'security_groups'] + sg_base_path = sg_base_path + \ + ['vpcs', vpc_id, 'security_groups'] else: sg_base_path = copy.deepcopy(current_path[0:6]) sg_base_path[1] = 'ec2' @@ -474,20 +512,25 @@ def match_security_groups_and_resources_callback(self, current_config, path, cur manage_dictionary(sg, 'used_by', {}) manage_dictionary(sg['used_by'], service, {}) manage_dictionary(sg['used_by'][service], 'resource_type', {}) - manage_dictionary(sg['used_by'][service]['resource_type'], resource_type, {} if resource_status else []) + manage_dictionary(sg['used_by'][service]['resource_type'], resource_type, { + } if resource_status else []) if resource_status: - manage_dictionary(sg['used_by'][service]['resource_type'][resource_type], resource_status, []) + manage_dictionary( + sg['used_by'][service]['resource_type'][resource_type], resource_status, []) if resource_id not in sg['used_by'][service]['resource_type'][resource_type][resource_status]: - sg['used_by'][service]['resource_type'][resource_type][resource_status].append(resource_id) + sg['used_by'][service]['resource_type'][resource_type][resource_status].append( + resource_id) else: - sg['used_by'][service]['resource_type'][resource_type].append(resource_id) + sg['used_by'][service]['resource_type'][resource_type].append( + resource_id) except Exception as e: region = current_path[3] vpc_id = current_path[5] if vpc_id == ec2_classic and resource_type == 'elbs': pass else: - print_error('Failed to parse %s in %s in %s' % (resource_type, vpc_id, region)) + print_error('Failed to parse %s in %s in %s' % + (resource_type, vpc_id, region)) print_exception(e) def _set_emr_vpc_ids(self): @@ -513,7 +556,8 @@ def set_emr_vpc_ids_callback(self, current_config, path, current_path, vpc_id, c elif 'RequestedEc2SubnetIds' in cluster['Ec2InstanceAttributes']: subnet_id = cluster['Ec2InstanceAttributes']['RequestedEc2SubnetIds'] else: - print_error('Unable to determine VPC id for EMR cluster %s' % str(cluster_id)) + print_error( + 'Unable to determine VPC id for EMR cluster %s' % str(cluster_id)) continue if sg_id in self.sg_map: vpc_id = self.sg_map[sg_id]['vpc_id'] @@ -527,7 +571,8 @@ def set_emr_vpc_ids_callback(self, current_config, path, current_path, vpc_id, c pop_list.append(cluster_id) sid_found = True if not sid_found: - print_error('Unable to determine VPC id for %s' % (str(subnet_id) if subnet_id else str(sg_id))) + print_error('Unable to determine VPC id for %s' % + (str(subnet_id) if subnet_id else str(sg_id))) continue if vpc_id: region_vpcs_config = get_object_at(current_path) @@ -541,7 +586,8 @@ def set_emr_vpc_ids_callback(self, current_config, path, current_path, vpc_id, c def sort_vpc_flow_logs_callback(self, current_config, path, current_path, flow_log_id, callback_args): attached_resource = current_config['ResourceId'] if attached_resource.startswith('vpc-'): - vpc_path = combine_paths(current_path[0:4], ['vpcs', attached_resource]) + vpc_path = combine_paths( + current_path[0:4], ['vpcs', attached_resource]) try: attached_vpc = get_object_at(self, vpc_path) except Exception: @@ -553,9 +599,11 @@ def sort_vpc_flow_logs_callback(self, current_config, path, current_path, flow_l if flow_log_id not in attached_vpc['flow_logs']: attached_vpc['flow_logs'].append(flow_log_id) for subnet_id in attached_vpc['subnets']: - manage_dictionary(attached_vpc['subnets'][subnet_id], 'flow_logs', []) + manage_dictionary( + attached_vpc['subnets'][subnet_id], 'flow_logs', []) if flow_log_id not in attached_vpc['subnets'][subnet_id]['flow_logs']: - attached_vpc['subnets'][subnet_id]['flow_logs'].append(flow_log_id) + attached_vpc['subnets'][subnet_id]['flow_logs'].append( + flow_log_id) elif attached_resource.startswith('subnet-'): subnet_path = combine_paths(current_path[0:4], ['vpcs', self.subnet_map[attached_resource]['vpc_id'], 'subnets', @@ -565,7 +613,8 @@ def sort_vpc_flow_logs_callback(self, current_config, path, current_path, flow_l if flow_log_id not in subnet['flow_logs']: subnet['flow_logs'].append(flow_log_id) else: - print_error('Resource %s attached to flow logs is not handled' % attached_resource) + print_error( + 'Resource %s attached to flow logs is not handled' % attached_resource) def get_db_attack_surface(self, current_config, path, current_path, db_id, callback_args): service = current_path[1] @@ -577,15 +626,18 @@ def get_db_attack_surface(self, current_config, path, current_path, db_id, callb listeners = [current_config['Endpoint']['Port']] security_groups = current_config['VpcSecurityGroups'] self._security_group_to_attack_surface(service_config['external_attack_surface'], public_dns, - current_path, [g['VpcSecurityGroupId'] for g in security_groups], + current_path, [ + g['VpcSecurityGroupId'] for g in security_groups], listeners) elif 'ConfigurationEndpoint' in current_config: # TODO : get the proper addresss - public_dns = current_config['ConfigurationEndpoint']['Address'].replace('.cfg', '') + public_dns = current_config['ConfigurationEndpoint']['Address'].replace( + '.cfg', '') listeners = [current_config['ConfigurationEndpoint']['Port']] security_groups = current_config['SecurityGroups'] self._security_group_to_attack_surface(service_config['external_attack_surface'], public_dns, - current_path, [g['SecurityGroupId'] for g in security_groups], + current_path, [ + g['SecurityGroupId'] for g in security_groups], listeners) # TODO :: Get Redis endpoint information @@ -595,7 +647,8 @@ def get_lb_attack_surface(self, current_config, path, current_path, elb_id, call manage_dictionary(elb_config, 'external_attack_surface', {}) if current_path[1] == 'elbv2' and current_config['Type'] == 'network': # Network LBs do not have a security group, lookup listeners instead - manage_dictionary(elb_config['external_attack_surface'], public_dns, {'protocols': {}}) + manage_dictionary( + elb_config['external_attack_surface'], public_dns, {'protocols': {}}) for listener in current_config['listeners']: protocol = current_config['listeners'][listener]['Protocol'] manage_dictionary(elb_config['external_attack_surface'][public_dns]['protocols'], protocol, @@ -605,8 +658,10 @@ def get_lb_attack_surface(self, current_config, path, current_path, elb_id, call elb_config['external_attack_surface'][public_dns]['protocols'][protocol]['ports'][listener][ 'cidrs'].append({'CIDR': '0.0.0.0/0'}) elif current_path[1] == 'elbv2' and current_config['Scheme'] == 'internet-facing': - elb_config['external_attack_surface'][public_dns] = {'protocols': {}} - security_groups = [g['GroupId'] for g in current_config['security_groups']] + elb_config['external_attack_surface'][public_dns] = { + 'protocols': {}} + security_groups = [g['GroupId'] + for g in current_config['security_groups']] listeners = [] for listener in current_config['listeners']: listeners.append(listener) @@ -615,7 +670,8 @@ def get_lb_attack_surface(self, current_config, path, current_path, elb_id, call elif current_config['Scheme'] == 'internet-facing': # Classic ELbs do not have a security group, lookup listeners instead public_dns = current_config['DNSName'] - manage_dictionary(elb_config['external_attack_surface'], public_dns, {'protocols': {'TCP': {'ports': {}}}}) + manage_dictionary(elb_config['external_attack_surface'], public_dns, { + 'protocols': {'TCP': {'ports': {}}}}) for listener in current_config['listeners']: manage_dictionary(elb_config['external_attack_surface'][public_dns]['protocols']['TCP']['ports'], listener, {'cidrs': []}) @@ -637,7 +693,8 @@ def _security_group_to_attack_surface(self, attack_surface_config, public_ip, cu for p in ingress_rules['protocols']: for port in ingress_rules['protocols'][p]['ports']: if len(listeners) == 0 and 'cidrs' in ingress_rules['protocols'][p]['ports'][port]: - manage_dictionary(attack_surface_config[public_ip]['protocols'], p, {'ports': {}}) + manage_dictionary( + attack_surface_config[public_ip]['protocols'], p, {'ports': {}}) manage_dictionary(attack_surface_config[public_ip]['protocols'][p]['ports'], port, {'cidrs': []}) attack_surface_config[public_ip]['protocols'][p]['ports'][port]['cidrs'] += \ @@ -657,7 +714,8 @@ def _security_group_to_attack_surface(self, attack_surface_config, public_ip, cu for listener in listeners: if (port_min and port_max) and port_min < int(listener) < port_max and \ 'cidrs' in ingress_rules['protocols'][p]['ports'][port]: - manage_dictionary(attack_surface_config[public_ip]['protocols'], p, {'ports': {}}) + manage_dictionary( + attack_surface_config[public_ip]['protocols'], p, {'ports': {}}) manage_dictionary(attack_surface_config[public_ip]['protocols'][p]['ports'], str(listener), {'cidrs': []}) attack_surface_config[public_ip]['protocols'][p]['ports'][str(listener)]['cidrs'] += \ diff --git a/ScoutSuite/providers/base/provider.py b/ScoutSuite/providers/base/provider.py index 7260fda14..f8652f898 100644 --- a/ScoutSuite/providers/base/provider.py +++ b/ScoutSuite/providers/base/provider.py @@ -25,8 +25,7 @@ class BaseProvider(object): all cloud providers """ - def __init__(self, report_dir=None, timestamp=None, services=None, skipped_services=None, thread_config=4, - **kwargs): + def __init__(self, report_dir=None, timestamp=None, services=None, skipped_services=None, thread_config=4, **kwargs): """ :aws_account_id AWS account ID @@ -38,7 +37,6 @@ def __init__(self, report_dir=None, timestamp=None, services=None, skipped_servi services = [] if services is None else services skipped_services = [] if skipped_services is None else skipped_services - self.credentials = None self.last_run = None self.metadata = None diff --git a/ScoutSuite/providers/gcp/provider.py b/ScoutSuite/providers/gcp/provider.py index 212a3adf4..e7a7bc701 100644 --- a/ScoutSuite/providers/gcp/provider.py +++ b/ScoutSuite/providers/gcp/provider.py @@ -43,91 +43,48 @@ def __init__(self, project_id=None, folder_id=None, organization_id=None, all_pr self.organization_id = organization_id self.services_config = GCPServicesConfig + self.credentials = kwargs['credentials'] + self._set_aws_account_id() super(GCPProvider, self).__init__(report_dir, timestamp, services, skipped_services, thread_config) + def authenticate(self, user_account=None, service_account=None, **kargs): - """ - Implement authentication for the GCP provider - Refer to https://google-auth.readthedocs.io/en/stable/reference/google.auth.html. + pass - :return: - """ + def _set_aws_account_id(self): + if self.all_projects: + if self.credentials.is_service_account and hasattr(self.credentials, 'service_account_email'): + self.aws_account_id = self.credentials.service_account_email # FIXME this is for AWS + else: + self.aws_account_id = 'GCP' # FIXME this is for AWS + self.profile = 'GCP' # FIXME this is for AWS - if user_account: - # disable GCP warning about using User Accounts - warnings.filterwarnings("ignore", "Your application has authenticated using end user credentials") - pass # Nothing more to do - elif service_account: - client_secrets_path = os.path.abspath(service_account) - os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = client_secrets_path - else: - print_error('Failed to authenticate to GCP - no supported account type') - return False + # Project passed through the CLI + elif self.project_id: + self.aws_account_id = self.project_id # FIXME this is for AWS + self.profile = self.project_id # FIXME this is for AWS - try: + # Folder passed through the CLI + elif self.folder_id: + self.aws_account_id = self.folder_id # FIXME this is for AWS + self.profile = self.folder_id # FIXME this is for AWS - self.credentials, project_id = google.auth.default() - if self.credentials: - - # All projects to which the user / Service Account has access to - if self.all_projects: - self.projects = self._get_projects(parent_type='all', - parent_id=None) - if service_account and hasattr(self.credentials, 'service_account_email'): - self.aws_account_id = self.credentials.service_account_email # FIXME this is for AWS - else: - self.aws_account_id = 'GCP' # FIXME this is for AWS - self.profile = 'GCP' # FIXME this is for AWS - # Project passed through the CLI - elif self.project_id: - self.projects = self._get_projects(parent_type='project', - parent_id=self.project_id) - self.aws_account_id = self.project_id # FIXME this is for AWS - self.profile = self.project_id # FIXME this is for AWS - - # Folder passed through the CLI - elif self.folder_id: - self.projects = self._get_projects(parent_type='folder', - parent_id=self.folder_id) - self.aws_account_id = self.folder_id # FIXME this is for AWS - self.profile = self.folder_id # FIXME this is for AWS - - # Organization passed through the CLI - elif self.organization_id: - self.projects = self._get_projects(parent_type='organization', - parent_id=self.organization_id) - self.aws_account_id = self.organization_id # FIXME this is for AWS - self.profile = self.organization_id # FIXME this is for AWS - - # Project inferred from default configuration - elif project_id: - self.projects = self._get_projects(parent_type='project', - parent_id=project_id) - self.aws_account_id = project_id # FIXME this is for AWS - self.profile = project_id # FIXME this is for AWS - - # Raise exception if none of the above - else: - print_info("Could not infer the Projects to scan and no default Project ID was found.") - return False - - # TODO this shouldn't be done here? but it has to in order to init with projects... - self.services.set_projects(projects=self.projects) - - return True - else: - return False + # Organization passed through the CLI + elif self.organization_id: + self.aws_account_id = self.organization_id # FIXME this is for AWS + self.profile = self.organization_id # FIXME this is for AWS - except google.auth.exceptions.DefaultCredentialsError as e: - print_error('Failed to authenticate to GCP') - print_exception(e) - return False + # # Project inferred from default configuration + # elif project_id: + # self.aws_account_id = project_id # FIXME this is for AWS + # self.profile = project_id # FIXME this is for AWS + + async def fetch(self, regions=None, skipped_regions=None, partition_name=None): + self._get_projects() + self.services.set_projects(projects=self.projects) + super(GCPProvider, self).fetch(regions, skipped_regions, partition_name) - except googleapiclient.errors.HttpError as e: - print_error('Failed to authenticate to GCP') - print_exception(e) - return False def preprocessing(self, ip_ranges=None, ip_ranges_name_key=None): """ @@ -145,7 +102,34 @@ def preprocessing(self, ip_ranges=None, ip_ranges_name_key=None): super(GCPProvider, self).preprocessing() - def _get_projects(self, parent_type, parent_id): + def _get_projects(self): + # All projects to which the user / Service Account has access to + if self.all_projects: + self.projects = self._gcp_facade_get_projects(parent_type='all', parent_id=None) + + # Project passed through the CLI + elif self.project_id: + self.projects = self._gcp_facade_get_projects(parent_type='project', parent_id=self.project_id) + + # Folder passed through the CLI + elif self.folder_id: + self.projects = self._gcp_facade_get_projects(parent_type='folder', parent_id=self.folder_id) + + # Organization passed through the CLI + elif self.organization_id: + self.projects = self._gcp_facade_get_projects(parent_type='organization', parent_id=self.organization_id) + + # # Project inferred from default configuration + # elif project_id: + # self.projects = self._gcp_facade_get_projects(parent_type='project', parent_id=project_id) + + # Raise exception if none of the above + else: + print_info("Could not infer the Projects to scan and no default Project ID was found.") + + + # TODO: This should be moved to the GCP facade + def _gcp_facade_get_projects(self, parent_type, parent_id): """ Returns all the projects in a given organization or folder. For a project_id it only returns the project details. @@ -169,8 +153,7 @@ def _get_projects(self, parent_type, parent_id): """ resource_manager_client_v1 = gcp_connect_service(service='cloudresourcemanager', credentials=self.credentials) - resource_manager_client_v2 = gcp_connect_service(service='cloudresourcemanager-v2', - credentials=self.credentials) + resource_manager_client_v2 = gcp_connect_service(service='cloudresourcemanager-v2', credentials=self.credentials) try: if parent_type == 'project': From a35043c4c50f39e5313a1869fa56079befa8823b Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 27 Mar 2019 11:53:19 -0400 Subject: [PATCH 611/667] Fixed default project issue --- .../providers/gcp/authentication_strategy.py | 7 ++++--- ScoutSuite/providers/gcp/provider.py | 16 ++++++++-------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/ScoutSuite/providers/gcp/authentication_strategy.py b/ScoutSuite/providers/gcp/authentication_strategy.py index 0871019d1..a9cce6b5b 100644 --- a/ScoutSuite/providers/gcp/authentication_strategy.py +++ b/ScoutSuite/providers/gcp/authentication_strategy.py @@ -6,7 +6,7 @@ from ScoutSuite.providers.base.authentication_strategy import AuthenticationStrategy, AuthenticationException class GCPAuthenticationStrategy(AuthenticationStrategy): - + def authenticate(self, user_account=None, service_account=None, **kargs): """ Implements authentication for the GCP provider @@ -24,9 +24,10 @@ def authenticate(self, user_account=None, service_account=None, **kargs): print_error('Failed to authenticate to GCP - no supported account type') raise AuthenticationException() - credentials, _ = google.auth.default() + credentials, default_project_id = google.auth.default() if not credentials: raise AuthenticationException() - + credentials.is_service_account = service_account is not None + credentials.default_project_id = default_project_id return credentials \ No newline at end of file diff --git a/ScoutSuite/providers/gcp/provider.py b/ScoutSuite/providers/gcp/provider.py index e7a7bc701..157c4ecfc 100644 --- a/ScoutSuite/providers/gcp/provider.py +++ b/ScoutSuite/providers/gcp/provider.py @@ -75,15 +75,15 @@ def _set_aws_account_id(self): self.aws_account_id = self.organization_id # FIXME this is for AWS self.profile = self.organization_id # FIXME this is for AWS - # # Project inferred from default configuration - # elif project_id: - # self.aws_account_id = project_id # FIXME this is for AWS - # self.profile = project_id # FIXME this is for AWS + # Project inferred from default configuration + elif self.credentials.default_project_id: + self.aws_account_id = self.credentials.default_project_id # FIXME this is for AWS + self.profile = self.credentials.default_project_id # FIXME this is for AWS async def fetch(self, regions=None, skipped_regions=None, partition_name=None): self._get_projects() self.services.set_projects(projects=self.projects) - super(GCPProvider, self).fetch(regions, skipped_regions, partition_name) + await super(GCPProvider, self).fetch(regions, skipped_regions, partition_name) def preprocessing(self, ip_ranges=None, ip_ranges_name_key=None): @@ -119,9 +119,9 @@ def _get_projects(self): elif self.organization_id: self.projects = self._gcp_facade_get_projects(parent_type='organization', parent_id=self.organization_id) - # # Project inferred from default configuration - # elif project_id: - # self.projects = self._gcp_facade_get_projects(parent_type='project', parent_id=project_id) + # Project inferred from default configuration + elif self.credentials.default_project_id: + self.projects = self._gcp_facade_get_projects(parent_type='project', parent_id=self.credentials.default_project_id) # Raise exception if none of the above else: From 009a2db1b131e3d0384a6d62fd899a204854b87a Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 27 Mar 2019 11:53:24 -0400 Subject: [PATCH 612/667] Lint --- ScoutSuite/providers/gcp/authentication_strategy.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ScoutSuite/providers/gcp/authentication_strategy.py b/ScoutSuite/providers/gcp/authentication_strategy.py index a9cce6b5b..d0fec3cc6 100644 --- a/ScoutSuite/providers/gcp/authentication_strategy.py +++ b/ScoutSuite/providers/gcp/authentication_strategy.py @@ -5,6 +5,7 @@ from ScoutSuite.core.console import print_error from ScoutSuite.providers.base.authentication_strategy import AuthenticationStrategy, AuthenticationException + class GCPAuthenticationStrategy(AuthenticationStrategy): def authenticate(self, user_account=None, service_account=None, **kargs): @@ -15,13 +16,15 @@ def authenticate(self, user_account=None, service_account=None, **kargs): if user_account: # disable GCP warning about using User Accounts - warnings.filterwarnings("ignore", "Your application has authenticated using end user credentials") + warnings.filterwarnings( + "ignore", "Your application has authenticated using end user credentials") pass # Nothing more to do elif service_account: client_secrets_path = os.path.abspath(service_account) os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = client_secrets_path else: - print_error('Failed to authenticate to GCP - no supported account type') + print_error( + 'Failed to authenticate to GCP - no supported account type') raise AuthenticationException() credentials, default_project_id = google.auth.default() @@ -30,4 +33,4 @@ def authenticate(self, user_account=None, service_account=None, **kargs): credentials.is_service_account = service_account is not None credentials.default_project_id = default_project_id - return credentials \ No newline at end of file + return credentials From 965196147eef1cf3a04322a64c46407b2debea40 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 27 Mar 2019 12:26:31 -0400 Subject: [PATCH 613/667] Implemented azure authentication strategy --- .../azure/authentication_strategy.py | 115 ++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 ScoutSuite/providers/azure/authentication_strategy.py diff --git a/ScoutSuite/providers/azure/authentication_strategy.py b/ScoutSuite/providers/azure/authentication_strategy.py new file mode 100644 index 000000000..d4e6b3624 --- /dev/null +++ b/ScoutSuite/providers/azure/authentication_strategy.py @@ -0,0 +1,115 @@ +import json + +from getpass import getpass + +from azure.common.credentials import ServicePrincipalCredentials, UserPassCredentials, get_azure_cli_credentials +from azure.mgmt.resource import SubscriptionClient +from msrestazure.azure_active_directory import MSIAuthentication + +from ScoutSuite.core.console import print_error, print_exception +from ScoutSuite.providers.base.authentication_strategy import AuthenticationStrategy, AuthenticationException + + +class AzureCredentials: + + def __init__(self, credentials, subscription_id, aws_account_id): + self.credentials = credentials + self.subscription_id = subscription_id + self.aws_account_id = aws_account_id + + +class AzureAuthenticationStrategy(AuthenticationStrategy): + + def authenticate(self, cli=None, msi=None, service_principal=None, file_auth=None, user_account=None, + tenant_id=None, subscription_id=None, client_id=None, client_secret=None, username=None, + password=None, **kargs): + """ + Implements authentication for the Azure provider + """ + try: + if cli: + # TODO: Remove aws_account_id + cli_credentials, aws_account_id = get_azure_cli_credentials() + credentials = AzureCredentials( + cli_credentials, aws_account_id, aws_account_id) + return credentials + + elif msi: + msi_auth_credentials = MSIAuthentication() + + # Get the subscription ID + subscription_client = SubscriptionClient(msi_auth_credentials) + + try: + # Tries to read the subscription list + subscription = next( + subscription_client.subscriptions.list()) + aws_account_id = subscription.subscription_id + + except StopIteration: + # If the VM cannot read subscription list, ask Subscription ID: + aws_account_id = input('Subscription ID: ') + + credentials = AzureCredentials( + credentials, aws_account_id, aws_account_id) + return credentials + + elif file_auth: + data = json.loads(file_auth.read()) + subscription_id = data.get('subscriptionId') + tenant_id = data.get('tenantId') + client_id = data.get('clientId') + client_secret = data.get('clientSecret') + + aws_account_id = tenant_id # TODO this is for AWS + + credentials = ServicePrincipalCredentials( + client_id=client_id, + secret=client_secret, + tenant=tenant_id + ) + + return AzureCredentials(credentials, subscription_id, aws_account_id) + + elif service_principal: + subscription_id = subscription_id if subscription_id else input( + "Subscription ID: ") + tenant_id = tenant_id if tenant_id else input("Tenant ID: ") + client_id = client_id if client_id else input("Client ID: ") + client_secret = client_secret if client_secret else getpass( + "Client secret: ") + + credentials = ServicePrincipalCredentials( + client_id=client_id, + secret=client_secret, + tenant=tenant_id + ) + + return AzureCredentials(credentials, subscription_id, tenant_id) + + elif user_account: + username = username if username else input("Username: ") + password = password if password else getpass("Password: ") + + credentials = UserPassCredentials(username, password) + + if subscription_id: + aws_account_id = subscription_id + else: + # Get the subscription ID + subscription_client = SubscriptionClient(credentials) + try: + # Tries to read the subscription list + subscription = next( + subscription_client.subscriptions.list()) + aws_account_id = subscription.subscription_id + except StopIteration: + # If the user cannot read subscription list, ask Subscription ID: + aws_account_id = input('Subscription ID: ') + + return AzureCredentials(credentials, aws_account_id, aws_account_id) + + except Exception as e: + print_error('Failed to authenticate to Azure') + print_exception(e) + return False From 1c7db1404019e5d922590750a8196102bf3b2df9 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 27 Mar 2019 12:26:43 -0400 Subject: [PATCH 614/667] Integrated azure authentication strategy --- ScoutSuite/providers/azure/provider.py | 16 +++------------- .../base/authentication_strategy_factory.py | 4 +++- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/ScoutSuite/providers/azure/provider.py b/ScoutSuite/providers/azure/provider.py index 295d0aeac..370795eea 100644 --- a/ScoutSuite/providers/azure/provider.py +++ b/ScoutSuite/providers/azure/provider.py @@ -1,25 +1,12 @@ # -*- coding: utf-8 -*- import os -import json - -from getpass import getpass from ScoutSuite.core.console import print_error, print_exception from ScoutSuite.providers.base.provider import BaseProvider from ScoutSuite.providers.azure.configs.services import AzureServicesConfig -from msrestazure.azure_active_directory import MSIAuthentication -from azure.mgmt.resource import SubscriptionClient -from azure.common.credentials import ServicePrincipalCredentials, UserPassCredentials, get_azure_cli_credentials - - -class AzureCredentials: - - def __init__(self, credentials, subscription_id): - self.credentials = credentials - self.subscription_id = subscription_id class AzureProvider(BaseProvider): @@ -40,6 +27,9 @@ def __init__(self, project_id=None, organization_id=None, self.provider_name = 'Microsoft Azure' self.services_config = AzureServicesConfig + + self.credentials = kwargs['credentials'] + self.aws_account_id = self.credentials.aws_account_id # TODO : Get rid of aws_account_id super(AzureProvider, self).__init__(report_dir, timestamp, services, skipped_services, thread_config) diff --git a/ScoutSuite/providers/base/authentication_strategy_factory.py b/ScoutSuite/providers/base/authentication_strategy_factory.py index d60548923..c79da9f83 100644 --- a/ScoutSuite/providers/base/authentication_strategy_factory.py +++ b/ScoutSuite/providers/base/authentication_strategy_factory.py @@ -1,9 +1,11 @@ from ScoutSuite.providers.aws.authentication_strategy import AWSAuthenticationStrategy from ScoutSuite.providers.gcp.authentication_strategy import GCPAuthenticationStrategy +from ScoutSuite.providers.azure.authentication_strategy import AzureAuthenticationStrategy _strategies = { 'aws': AWSAuthenticationStrategy, - 'gcp': GCPAuthenticationStrategy + 'gcp': GCPAuthenticationStrategy, + 'azure': AzureAuthenticationStrategy } From 31b2889f3c66f9a38300b9b20eeac800f73a23d5 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 27 Mar 2019 12:27:03 -0400 Subject: [PATCH 615/667] Lint --- ScoutSuite/providers/gcp/provider.py | 51 +++++++++++++++------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/ScoutSuite/providers/gcp/provider.py b/ScoutSuite/providers/gcp/provider.py index 157c4ecfc..92f590b1d 100644 --- a/ScoutSuite/providers/gcp/provider.py +++ b/ScoutSuite/providers/gcp/provider.py @@ -12,13 +12,6 @@ from ScoutSuite.providers.gcp.utils import gcp_connect_service -class GCPCredentials(): - - def __init__(self, api_client_credentials, cloud_client_credentials): - self.api_client_credentials = api_client_credentials - self.cloud_client_credentials = cloud_client_credentials - - class GCPProvider(BaseProvider): """ Implements provider for GCP @@ -31,7 +24,8 @@ def __init__(self, project_id=None, folder_id=None, organization_id=None, all_pr self.profile = 'gcp-profile' # TODO this is aws-specific - self.metadata_path = '%s/metadata.json' % os.path.split(os.path.abspath(__file__))[0] + self.metadata_path = '%s/metadata.json' % os.path.split( + os.path.abspath(__file__))[0] self.provider_code = 'gcp' self.provider_name = 'Google Cloud Platform' @@ -46,8 +40,8 @@ def __init__(self, project_id=None, folder_id=None, organization_id=None, all_pr self.credentials = kwargs['credentials'] self._set_aws_account_id() - super(GCPProvider, self).__init__(report_dir, timestamp, services, skipped_services, thread_config) - + super(GCPProvider, self).__init__(report_dir, timestamp, + services, skipped_services, thread_config) def authenticate(self, user_account=None, service_account=None, **kargs): pass @@ -85,7 +79,6 @@ async def fetch(self, regions=None, skipped_regions=None, partition_name=None): self.services.set_projects(projects=self.projects) await super(GCPProvider, self).fetch(regions, skipped_regions, partition_name) - def preprocessing(self, ip_ranges=None, ip_ranges_name_key=None): """ TODO description @@ -105,30 +98,36 @@ def preprocessing(self, ip_ranges=None, ip_ranges_name_key=None): def _get_projects(self): # All projects to which the user / Service Account has access to if self.all_projects: - self.projects = self._gcp_facade_get_projects(parent_type='all', parent_id=None) + self.projects = self._gcp_facade_get_projects( + parent_type='all', parent_id=None) # Project passed through the CLI elif self.project_id: - self.projects = self._gcp_facade_get_projects(parent_type='project', parent_id=self.project_id) + self.projects = self._gcp_facade_get_projects( + parent_type='project', parent_id=self.project_id) # Folder passed through the CLI elif self.folder_id: - self.projects = self._gcp_facade_get_projects(parent_type='folder', parent_id=self.folder_id) + self.projects = self._gcp_facade_get_projects( + parent_type='folder', parent_id=self.folder_id) # Organization passed through the CLI elif self.organization_id: - self.projects = self._gcp_facade_get_projects(parent_type='organization', parent_id=self.organization_id) + self.projects = self._gcp_facade_get_projects( + parent_type='organization', parent_id=self.organization_id) # Project inferred from default configuration elif self.credentials.default_project_id: - self.projects = self._gcp_facade_get_projects(parent_type='project', parent_id=self.credentials.default_project_id) + self.projects = self._gcp_facade_get_projects( + parent_type='project', parent_id=self.credentials.default_project_id) # Raise exception if none of the above else: - print_info("Could not infer the Projects to scan and no default Project ID was found.") - + print_info( + "Could not infer the Projects to scan and no default Project ID was found.") # TODO: This should be moved to the GCP facade + def _gcp_facade_get_projects(self, parent_type, parent_id): """ Returns all the projects in a given organization or folder. For a project_id it only returns the project @@ -152,12 +151,15 @@ def _gcp_facade_get_projects(self, parent_type, parent_id): projects.append(p.project_id) """ - resource_manager_client_v1 = gcp_connect_service(service='cloudresourcemanager', credentials=self.credentials) - resource_manager_client_v2 = gcp_connect_service(service='cloudresourcemanager-v2', credentials=self.credentials) + resource_manager_client_v1 = gcp_connect_service( + service='cloudresourcemanager', credentials=self.credentials) + resource_manager_client_v2 = gcp_connect_service( + service='cloudresourcemanager-v2', credentials=self.credentials) try: if parent_type == 'project': - project_response = resource_manager_client_v1.projects().list(filter='id:%s' % parent_id).execute() + project_response = resource_manager_client_v1.projects().list(filter='id:%s' % + parent_id).execute() if 'projects' in project_response.keys(): for project in project_response['projects']: if project['lifecycleState'] == "ACTIVE": @@ -172,7 +174,8 @@ def _gcp_facade_get_projects(self, parent_type, parent_id): else: # get parent children projects - request = resource_manager_client_v1.projects().list(filter='parent.id:%s' % parent_id) + request = resource_manager_client_v1.projects().list( + filter='parent.id:%s' % parent_id) while request is not None: response = request.execute() @@ -189,7 +192,8 @@ def _gcp_facade_get_projects(self, parent_type, parent_id): parent='%ss/%s' % (parent_type, parent_id)).execute() if 'folders' in folder_response.keys(): for folder in folder_response['folders']: - projects.extend(self._get_projects("folder", folder['name'].strip(u'folders/'))) + projects.extend(self._get_projects( + "folder", folder['name'].strip(u'folders/'))) print_info("Found {} project(s) to scan.".format(len(projects))) @@ -200,7 +204,6 @@ def _gcp_facade_get_projects(self, parent_type, parent_id): finally: return projects - def _match_instances_and_snapshots(self): """ Compare Compute Engine instances and snapshots to identify instance disks that do not have a snapshot. From e598c25c6d468abec68c75c8e547d2a2c676af23 Mon Sep 17 00:00:00 2001 From: misg Date: Wed, 27 Mar 2019 15:27:04 -0400 Subject: [PATCH 616/667] Define _fetch_children_of_all_resources method and update docstrings. --- .../providers/aws/resources/resources.py | 63 ++++++++++++------- 1 file changed, 41 insertions(+), 22 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/resources.py b/ScoutSuite/providers/aws/resources/resources.py index ca87905ec..d8ac5323d 100644 --- a/ScoutSuite/providers/aws/resources/resources.py +++ b/ScoutSuite/providers/aws/resources/resources.py @@ -4,7 +4,6 @@ import asyncio from ScoutSuite.providers.base.configs.resources import Resources, CompositeResources -from ScoutSuite.providers.aws.facade.facade import AWSFacade class AWSResources(Resources, metaclass=abc.ABCMeta): @@ -13,9 +12,9 @@ class AWSResources(Resources, metaclass=abc.ABCMeta): def __init__(self, facade, scope: dict): """ - :param scope: The scope holds the scope in which the resource is located. This usually means \ - at least a region, but can also contain a VPC id, an owner id, etc. It should be \ - used when fetching the data through the facade. + :param scope: The scope holds the scope in which the resource is located. This usually means at least a region, + but can also contain a VPC id, an owner id, etc. It should be used when fetching the data through + the facade. """ self.scope = scope @@ -24,29 +23,49 @@ def __init__(self, facade, scope: dict): class AWSCompositeResources(AWSResources, CompositeResources, metaclass=abc.ABCMeta): - """This class represents a collection of AWSResources. Classes extending AWSCompositeResources should \ - define a "_children" attribute which consists of a list of tuples describing the children. The tuples \ - are expected to respect the following format: (, ). The child_name is used by \ - indicates the name under which the child will be stored in the parent object. + """This class represents a collection of AWSResources. Classes extending AWSCompositeResources should define a + "_children" attribute which consists of a list of tuples describing the children. The tuples are expected to + respect the following format: (, ). The child_name is used by indicates the name under + which the child will be stored in the parent object. """ - - async def _fetch_children(self, parent: object, scope: dict): - """This method calls fetch_all on each child defined in "_children" and stores the fetched resources \ - in the parent under the key associated with the child. It also creates a "_count" entry \ - for each child. - :param parent: The object in which the children should be stored - :param scope: The scope passed to the children constructors + async def _fetch_children_of_all_resources(self, resources: dict, scopes: dict): + """ This method iterates through a collection of resources and fetches all children of each resource, in a + concurrent way. + + :param resources: list of (composite) resources + :param scopes: dict that maps resource parent keys to scopes (dict) that should be used to retrieve children + of each resource. + """ + if len(resources) == 0: + return + + tasks = { + asyncio.ensure_future( + self._fetch_children(resource_parent=resource_parent, scope=scopes[resource_parent_key]) + ) for (resource_parent_key, resource_parent) in resources.items() + } + await asyncio.wait(tasks) + + async def _fetch_children(self, resource_parent: object, scope: dict): + """This method fetches all children of a given resource (the so called 'resource_parent') by calling fetch_all + method on each child defined in '_children' and then stores the fetched resources in `resource_parent` under + the key associated with the child. It also creates a "_count" entry for each child. + + :param resource_parent: The resource in which the children will be stored. + :param scope: The scope passed to the children constructors. """ children = [(child_class(self.facade, scope), child_name) for (child_class, child_name) in self._children] - # fetch all children concurrently: - await asyncio.wait({asyncio.ensure_future(child.fetch_all()) for (child, _) in children}) - # update parent content: + # Fetch all children concurrently: + await asyncio.wait( + {asyncio.ensure_future(child.fetch_all()) for (child, _) in children} + ) + # Update parent content: for child, child_name in children: - if parent.get(child_name) is None: - parent[child_name] = {} + if resource_parent.get(child_name) is None: + resource_parent[child_name] = {} - parent[child_name].update(child) - parent[child_name + '_count'] = len(child) + resource_parent[child_name].update(child) + resource_parent[child_name + '_count'] = len(child) From ca43815b6a5e6657b785acf8985f94b95b51a999 Mon Sep 17 00:00:00 2001 From: misg Date: Wed, 27 Mar 2019 15:28:38 -0400 Subject: [PATCH 617/667] Move fetching of the load balancer attributes to the facade. --- ScoutSuite/providers/aws/facade/elbv2.py | 21 +++++++++++++------ .../aws/resources/elbv2/load_balancers.py | 21 +++++++++---------- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/elbv2.py b/ScoutSuite/providers/aws/facade/elbv2.py index b46603d5d..62538bcfc 100644 --- a/ScoutSuite/providers/aws/facade/elbv2.py +++ b/ScoutSuite/providers/aws/facade/elbv2.py @@ -1,10 +1,10 @@ +import asyncio + from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade from ScoutSuite.providers.aws.utils import ec2_classic from ScoutSuite.providers.utils import run_concurrently -from asyncio import Lock - class ELBv2Facade(AWSBaseFacade): regional_load_balancers_cache_locks = {} @@ -15,7 +15,7 @@ async def get_load_balancers(self, region: str, vpc: str): return [load_balancer for load_balancer in self.load_balancers_cache[region] if load_balancer['VpcId'] == vpc] async def cache_load_balancers(self, region): - async with self.regional_load_balancers_cache_locks.setdefault(region, Lock()): + async with self.regional_load_balancers_cache_locks.setdefault(region, asyncio.Lock()): if region in self.load_balancers_cache: return @@ -27,11 +27,20 @@ async def cache_load_balancers(self, region): load_balancer['VpcId'] =\ load_balancer['VpcId'] if 'VpcId' in load_balancer and load_balancer['VpcId'] else ec2_classic - async def get_load_balancer_attributes(self, region: str, load_balancer_arn: str): + if len(self.load_balancers_cache[region]) == 0: + return + tasks = { + asyncio.ensure_future( + self.get_and_set_load_balancer_attributes(region, load_balancer) + ) for load_balancer in self.load_balancers_cache[region] + } + await asyncio.wait(tasks) + + async def get_and_set_load_balancer_attributes(self, region: str, load_balancer: dict): elbv2_client = AWSFacadeUtils.get_client('elbv2', self.session, region) - return await run_concurrently( + load_balancer['attributes'] = await run_concurrently( lambda: elbv2_client.describe_load_balancer_attributes( - LoadBalancerArn=load_balancer_arn)['Attributes'] + LoadBalancerArn=load_balancer['LoadBalancerArn'])['Attributes'] ) async def get_listeners(self, region: str, load_balancer_arn: str): diff --git a/ScoutSuite/providers/aws/resources/elbv2/load_balancers.py b/ScoutSuite/providers/aws/resources/elbv2/load_balancers.py index e67ca7f5a..ace1606a4 100644 --- a/ScoutSuite/providers/aws/resources/elbv2/load_balancers.py +++ b/ScoutSuite/providers/aws/resources/elbv2/load_balancers.py @@ -1,6 +1,5 @@ from ScoutSuite.providers.aws.resources.resources import AWSCompositeResources from ScoutSuite.providers.utils import get_non_provider_id - from .listeners import Listeners @@ -10,15 +9,18 @@ class LoadBalancers(AWSCompositeResources): ] async def fetch_all(self, **kwargs): - raw_loads_balancers = await self.facade.elbv2.get_load_balancers(self.scope['region'], self.scope['vpc']) - # TODO: parallelize the following loop which is async: - for raw_load_balancer in raw_loads_balancers: - id, load_balancer = await self._parse_load_balancer(raw_load_balancer) + raw_load_balancers = await self.facade.elbv2.get_load_balancers(self.scope['region'], self.scope['vpc']) + for raw_load_balancer in raw_load_balancers: + id, load_balancer = self._parse_load_balancer(raw_load_balancer) self[id] = load_balancer - await self._fetch_children( - parent=load_balancer, scope={'region': self.scope['region'], 'load_balancer_arn': load_balancer['arn']}) - async def _parse_load_balancer(self, load_balancer): + await self._fetch_children_of_all_resources( + resources=self, + scopes={load_balancer_id: {'region': self.scope['region'], 'load_balancer_arn': load_balancer['arn']} + for (load_balancer_id, load_balancer) in self.items()} + ) + + def _parse_load_balancer(self, load_balancer): load_balancer['arn'] = load_balancer.pop('LoadBalancerArn') load_balancer['name'] = load_balancer.pop('LoadBalancerName') load_balancer['security_groups'] = [] @@ -28,7 +30,4 @@ async def _parse_load_balancer(self, load_balancer): load_balancer['security_groups'].append({'GroupId': sg}) load_balancer.pop('SecurityGroups') - load_balancer['attributes'] =\ - await self.facade.elbv2.get_load_balancer_attributes(self.scope['region'], load_balancer['arn']) - return get_non_provider_id(load_balancer['name']), load_balancer From 795874e084438d663bb1829d314cc5842a5b4031 Mon Sep 17 00:00:00 2001 From: misg Date: Wed, 27 Mar 2019 15:29:28 -0400 Subject: [PATCH 618/667] Parallelize and factorize async loops. --- .../redshift/cluster_parameter_groups.py | 12 +++++---- ScoutSuite/providers/aws/resources/regions.py | 16 +++-------- .../providers/aws/resources/ses/identities.py | 11 ++++---- .../providers/aws/resources/sns/topics.py | 17 +++++++----- ScoutSuite/providers/aws/resources/vpcs.py | 27 +++++++------------ 5 files changed, 37 insertions(+), 46 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/redshift/cluster_parameter_groups.py b/ScoutSuite/providers/aws/resources/redshift/cluster_parameter_groups.py index cd0240594..167a35a60 100644 --- a/ScoutSuite/providers/aws/resources/redshift/cluster_parameter_groups.py +++ b/ScoutSuite/providers/aws/resources/redshift/cluster_parameter_groups.py @@ -11,15 +11,17 @@ class ClusterParameterGroups(AWSCompositeResources): async def fetch_all(self, **kwargs): raw_parameter_groups = await self.facade.redshift.get_cluster_parameter_groups(self.scope['region']) - # TODO: parallelize this async loop: for raw_parameter_group in raw_parameter_groups: id, parameter_group = self._parse_parameter_group(raw_parameter_group) - await self._fetch_children( - parent=parameter_group, - scope={'region': self.scope['region'], 'parameter_group_name': parameter_group['name']} - ) self[id] = parameter_group + await self._fetch_children_of_all_resources( + resources=self, + scopes={parameter_group_id: {'region': self.scope['region'], + 'parameter_group_name': parameter_group['name']} + for (parameter_group_id, parameter_group) in self.items()} + ) + def _parse_parameter_group(self, raw_parameter_group): name = raw_parameter_group.pop('ParameterGroupName') id = get_non_provider_id(name) diff --git a/ScoutSuite/providers/aws/resources/regions.py b/ScoutSuite/providers/aws/resources/regions.py index 484a1404b..b450cfabf 100644 --- a/ScoutSuite/providers/aws/resources/regions.py +++ b/ScoutSuite/providers/aws/resources/regions.py @@ -25,18 +25,10 @@ async def fetch_all(self, credentials, regions=None, partition_name='aws'): 'name': region } - # TODO: make a refactoring of the following: - if len(self['regions']) == 0: - return - tasks = { - asyncio.ensure_future( - self._fetch_children( - self['regions'][region], - {'region': region, 'owner_id': account_id} - ) - ) for region in self['regions'] - } - await asyncio.wait(tasks) + await self._fetch_children_of_all_resources( + resources=self['regions'], + scopes={region: {'region': region, 'owner_id': account_id} for region in self['regions']} + ) self._set_counts() diff --git a/ScoutSuite/providers/aws/resources/ses/identities.py b/ScoutSuite/providers/aws/resources/ses/identities.py index 3f5bb6c8e..5f9dabd15 100644 --- a/ScoutSuite/providers/aws/resources/ses/identities.py +++ b/ScoutSuite/providers/aws/resources/ses/identities.py @@ -11,15 +11,16 @@ class Identities(AWSCompositeResources): async def fetch_all(self, **kwargs): raw_identities = await self.facade.ses.get_identities(self.scope['region']) - # TODO: parallelize the following async loop: for raw_identity in raw_identities: id, identity = self._parse_identity(raw_identity) - await self._fetch_children( - parent=identity, - scope={'region': self.scope['region'], 'identity_name': identity['name']} - ) self[id] = identity + await self._fetch_children_of_all_resources( + resources=self, + scopes={identity_id: {'region': self.scope['region'], 'identity_name': identity['name']} + for (identity_id, identity) in self.items()} + ) + def _parse_identity(self, raw_identity): identity_name, dkim_attributes = raw_identity identity = {} diff --git a/ScoutSuite/providers/aws/resources/sns/topics.py b/ScoutSuite/providers/aws/resources/sns/topics.py index e15740ac0..57cc51d2f 100644 --- a/ScoutSuite/providers/aws/resources/sns/topics.py +++ b/ScoutSuite/providers/aws/resources/sns/topics.py @@ -12,17 +12,20 @@ class Topics(AWSCompositeResources): async def fetch_all(self, **kwargs): raw_topics = await self.facade.sns.get_topics(self.scope['region']) - # TODO: parallelize this async loop: for raw_topic in raw_topics: topic_name, topic = self._parse_topic(raw_topic) - await self._fetch_children( - parent=topic, - scope={'region': self.scope['region'], 'topic_name': topic_name} - ) - # Fix subscriptions count: - topic['subscriptions_count'] = topic['subscriptions'].pop('subscriptions_count') self[topic_name] = topic + await self._fetch_children_of_all_resources( + resources=self, + scopes={topic_id: {'region': self.scope['region'], 'topic_name': topic['name']} + for (topic_id, topic) in self.items()} + ) + + # Fix subscriptions count: + for topic in self.values(): + topic['subscriptions_count'] = topic['subscriptions'].pop('subscriptions_count') + def _parse_topic(self, raw_topic): raw_topic['arn'] = raw_topic.pop('TopicArn') raw_topic['name'] = raw_topic['arn'].split(':')[-1] diff --git a/ScoutSuite/providers/aws/resources/vpcs.py b/ScoutSuite/providers/aws/resources/vpcs.py index 6443c6466..9dccc0e5a 100644 --- a/ScoutSuite/providers/aws/resources/vpcs.py +++ b/ScoutSuite/providers/aws/resources/vpcs.py @@ -1,5 +1,3 @@ -import asyncio - from ScoutSuite.providers.aws.resources.resources import AWSCompositeResources @@ -14,21 +12,16 @@ def __init__(self, facade, scope: dict, add_ec2_classic=False): self.add_ec2_classic = add_ec2_classic async def fetch_all(self, **kwargs): - vpcs = await self.facade.ec2.get_vpcs(self.scope['region']) - for vpc in vpcs: - name, resource = self._parse_vpc(vpc) - self[name] = resource - - if len(self) == 0: - return - - tasks = { - asyncio.ensure_future( - self._fetch_children(self[vpc], {'region': self.scope['region'], 'vpc': vpc}) - ) for vpc in self - } - - await asyncio.wait(tasks) + raw_vpcs = await self.facade.ec2.get_vpcs(self.scope['region']) + for raw_vpc in raw_vpcs: + vpc_id, vpc = self._parse_vpc(raw_vpc) + self[vpc_id] = vpc + + await self._fetch_children_of_all_resources( + resources=self, + scopes={vpc_id: {'region': self.scope['region'], 'vpc': vpc_id} + for vpc_id in self} + ) def _parse_vpc(self, vpc): return vpc['VpcId'], {} From d8d69c7754b6678f478124dcf098fbec317e6d2e Mon Sep 17 00:00:00 2001 From: misg Date: Wed, 27 Mar 2019 15:30:16 -0400 Subject: [PATCH 619/667] Define _fetch_children_of_all_resources method and add docstrings. --- .../providers/azure/resources/resources.py | 49 ++++++++++++++++--- 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/resources.py b/ScoutSuite/providers/azure/resources/resources.py index 5523bd4e7..438a149b0 100644 --- a/ScoutSuite/providers/azure/resources/resources.py +++ b/ScoutSuite/providers/azure/resources/resources.py @@ -1,19 +1,54 @@ +"""This module provides implementations for Resources and CompositeResources for Azure.""" + import abc import asyncio from ScoutSuite.providers.base.configs.resources import CompositeResources -# TODO: add docstrings. class AzureCompositeResources(CompositeResources, metaclass=abc.ABCMeta): - async def _fetch_children(self, parent, **kwargs): + """This class represents a collection of composite Resources (resources that include nested resources referred as + their children). Classes extending AzureCompositeResources have to define a '_children' attribute which consists of + a list of tuples describing the children. The tuples are expected to respect the following format: + (, ). 'child_name' is used to indicate the name under which the child resources will be + stored in the parent object. + """ + + async def _fetch_children_of_all_resources(self, resources: dict, kwargs: dict): + """This method iterates through a collection of resources and fetches all children of each resource, in a + concurrent way. + + :param resources: list of (composite) resources + :param kwargs: dict that maps resource parent keys to each kwargs (dict) used to retrieve child resources. + """ + if len(resources) == 0: + return + + tasks = { + asyncio.ensure_future( + self._fetch_children(resource_parent=resource_parent, **kwargs[resource_parent_key]) + ) for (resource_parent_key, resource_parent) in resources.items() + } + await asyncio.wait(tasks) + + async def _fetch_children(self, resource_parent, **kwargs): + """This method fetches all children of a given resource (the so called 'resource_parent') by calling fetch_all + method on each child defined in '_children' and then stores the fetched resources in `resource_parent` under + the key associated with the child. + + :param resource_parent: The resource in which the children will be stored. + :param kwargs: parameters that depend on the type of child resources, used to fetch them. + """ + children = [(child_class(**kwargs), child_name) for (child_class, child_name) in self._children] - # fetch all children concurrently: - await asyncio.wait({asyncio.ensure_future(child.fetch_all()) for (child, _) in children}) - # update parent content: + # Fetch all children concurrently: + await asyncio.wait( + {asyncio.ensure_future(child.fetch_all()) for (child, _) in children} + ) + # Update parent content: for child, name in children: if name: - parent[name] = child + resource_parent[name] = child else: - parent.update(child) + resource_parent.update(child) From ba75c85d5aca1ab7f8bec26c76781f9fd2c5bba5 Mon Sep 17 00:00:00 2001 From: misg Date: Wed, 27 Mar 2019 15:30:42 -0400 Subject: [PATCH 620/667] Fix parameter names. --- ScoutSuite/providers/azure/resources/network/networks.py | 2 +- .../providers/azure/resources/securitycenter/security_center.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/network/networks.py b/ScoutSuite/providers/azure/resources/network/networks.py index e872d5b44..7c4049a98 100644 --- a/ScoutSuite/providers/azure/resources/network/networks.py +++ b/ScoutSuite/providers/azure/resources/network/networks.py @@ -15,7 +15,7 @@ async def fetch_all(self, credentials, **kwargs): # TODO: build that facade somewhere else: facade = NetworkFacade(credentials.credentials, credentials.subscription_id) - await self._fetch_children(parent=self, facade=facade) + await self._fetch_children(resource_parent=self, facade=facade) self['network_security_groups_count'] = len(self['network_security_groups']) self['network_watchers_count'] = len(self['network_watchers']) diff --git a/ScoutSuite/providers/azure/resources/securitycenter/security_center.py b/ScoutSuite/providers/azure/resources/securitycenter/security_center.py index 6b9109f2e..eb07faabf 100644 --- a/ScoutSuite/providers/azure/resources/securitycenter/security_center.py +++ b/ScoutSuite/providers/azure/resources/securitycenter/security_center.py @@ -17,7 +17,7 @@ async def fetch_all(self, credentials, **kwargs): # TODO: build that facade somewhere else: facade = SecurityCenterFacade(credentials.credentials, credentials.subscription_id) - await self._fetch_children(parent=self, facade=facade) + await self._fetch_children(resource_parent=self, facade=facade) self['auto_provisioning_settings_count'] = len(self['auto_provisioning_settings']) self['pricings_count'] = len(self['pricings']) From 7bb879a0983463e6dce50462ad3490114fc24f38 Mon Sep 17 00:00:00 2001 From: misg Date: Wed, 27 Mar 2019 15:31:21 -0400 Subject: [PATCH 621/667] Factorize async loops. --- .../azure/resources/sqldatabase/databases.py | 22 +++++----------- .../azure/resources/sqldatabase/servers.py | 23 +++++----------- .../storageaccounts/storageaccounts.py | 26 ++++++------------- 3 files changed, 22 insertions(+), 49 deletions(-) diff --git a/ScoutSuite/providers/azure/resources/sqldatabase/databases.py b/ScoutSuite/providers/azure/resources/sqldatabase/databases.py index c51de1c92..ac4000a56 100644 --- a/ScoutSuite/providers/azure/resources/sqldatabase/databases.py +++ b/ScoutSuite/providers/azure/resources/sqldatabase/databases.py @@ -32,18 +32,10 @@ async def fetch_all(self): 'name': db.name } - # TODO: make a refactoring of the following: - if len(self) == 0: - return - tasks = { - asyncio.ensure_future( - self._fetch_children( - parent=db, - resource_group_name=self.resource_group_name, - server_name=self.server_name, - database_name=db['name'], - facade=self.facade - ) - ) for db in self.values() - } - await asyncio.wait(tasks) + await self._fetch_children_of_all_resources( + resources=self, + kwargs={db_id: {'resource_group_name': self.resource_group_name, + 'server_name': self.server_name, + 'database_name': db['name'], + 'facade': self.facade} for (db_id, db) in self.items()} + ) diff --git a/ScoutSuite/providers/azure/resources/sqldatabase/servers.py b/ScoutSuite/providers/azure/resources/sqldatabase/servers.py index f2f62e807..a46f698d9 100644 --- a/ScoutSuite/providers/azure/resources/sqldatabase/servers.py +++ b/ScoutSuite/providers/azure/resources/sqldatabase/servers.py @@ -33,20 +33,11 @@ async def fetch_all(self, credentials, **kwargs): 'name': server.name, 'resource_group_name': resource_group_name } - - # TODO: make a refactoring of the following: - if len(self['servers']) == 0: - return - tasks = { - asyncio.ensure_future( - self._fetch_children( - parent=server, - resource_group_name=server['resource_group_name'], - server_name=server['name'], - facade=facade - ) - ) for server in self['servers'].values() - } - await asyncio.wait(tasks) - self['servers_count'] = len(self['servers']) + + await self._fetch_children_of_all_resources( + resources=self['servers'], + kwargs={server_id: {'resource_group_name': server['resource_group_name'], + 'server_name': server['name'], + 'facade': facade} for (server_id, server) in self['servers'].items()} + ) diff --git a/ScoutSuite/providers/azure/resources/storageaccounts/storageaccounts.py b/ScoutSuite/providers/azure/resources/storageaccounts/storageaccounts.py index 6ccd6134b..f0c4a2e43 100644 --- a/ScoutSuite/providers/azure/resources/storageaccounts/storageaccounts.py +++ b/ScoutSuite/providers/azure/resources/storageaccounts/storageaccounts.py @@ -1,5 +1,3 @@ -import asyncio - from ScoutSuite.providers.azure.resources.resources import AzureCompositeResources from ScoutSuite.providers.azure.utils import get_resource_group_name from ScoutSuite.providers.utils import get_non_provider_id @@ -21,24 +19,16 @@ async def fetch_all(self, credentials, **kwargs): for raw_storage_account in await facade.get_storage_accounts(): id, storage_account = self._parse(raw_storage_account) self['storage_accounts'][id] = storage_account - - # TODO: make a refactoring of the following: - if len(self) == 0: - return - tasks = { - asyncio.ensure_future( - self._fetch_children( - parent=storage_account, - resource_group_name=storage_account['resource_group_name'], - storage_account_name=storage_account['name'], - facade=facade - ) - ) for storage_account in self['storage_accounts'].values() - } - await asyncio.wait(tasks) - self['storage_accounts_count'] = len(self['storage_accounts']) + await self._fetch_children_of_all_resources( + resources=self['storage_accounts'], + kwargs={storage_account_id: {'resource_group_name': storage_account['resource_group_name'], + 'storage_account_name': storage_account['name'], + 'facade': facade} + for (storage_account_id, storage_account) in self['storage_accounts'].items()} + ) + def _parse(self, raw_storage_account): storage_account = {} raw_id = raw_storage_account.id From d38adea54442077cd87c611f03d3271662d23ffb Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 27 Mar 2019 15:44:15 -0400 Subject: [PATCH 622/667] Integrated new authentication implementation --- ScoutSuite/providers/aws/configs/services.py | 3 ++- ScoutSuite/providers/aws/facade/facade.py | 7 ++----- ScoutSuite/providers/base/configs/services.py | 2 +- ScoutSuite/providers/base/provider.py | 2 +- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index e43c0895c..e1f23db40 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -51,7 +51,8 @@ class AWSServicesConfig(BaseServicesConfig): :ivar sqs: SQS configuration """ - def __init__(self, metadata=None, thread_config=4, **kwargs): + def __init__(self, credentials=None, thread_config=4, **kwargs): + super(AWSServicesConfig, self).__init__(credentials, thread_config) super(AWSServicesConfig, self).__init__(metadata, thread_config) self.cloudwatch = CloudWatch() diff --git a/ScoutSuite/providers/aws/facade/facade.py b/ScoutSuite/providers/aws/facade/facade.py index c6049fc6b..2e84b0a95 100644 --- a/ScoutSuite/providers/aws/facade/facade.py +++ b/ScoutSuite/providers/aws/facade/facade.py @@ -34,8 +34,9 @@ class AWSFacade(AWSBaseFacade): - def __init__(self, credentials: dict = None): + def __init__(self, credentials = None): self._set_session(credentials) + self._instantiate_facades() async def build_region_list(self, service: str, chosen_regions=None, partition_name='aws'): service = 'ec2containerservice' if service == 'ecs' else service @@ -63,10 +64,6 @@ def _set_session(self, credentials: dict): self.session = boto3.session.Session(**session_params) - # TODO: This should only be done in the constructor. I put this here for now, because this method is currently - # called from outside, but it should not happen. - self._instantiate_facades() - def _instantiate_facades(self): self.ec2 = EC2Facade(self.session) self.awslambda = LambdaFacade(self.session) diff --git a/ScoutSuite/providers/base/configs/services.py b/ScoutSuite/providers/base/configs/services.py index 8fcf4729c..4ee4dca39 100644 --- a/ScoutSuite/providers/base/configs/services.py +++ b/ScoutSuite/providers/base/configs/services.py @@ -7,7 +7,7 @@ class BaseServicesConfig(object): - def __init__(self, metadata=None, thread_config=4): + def __init__(self, credentials=None, thread_config=4): pass def _is_provider(self, provider_name): diff --git a/ScoutSuite/providers/base/provider.py b/ScoutSuite/providers/base/provider.py index f8652f898..b081be713 100644 --- a/ScoutSuite/providers/base/provider.py +++ b/ScoutSuite/providers/base/provider.py @@ -42,7 +42,7 @@ def __init__(self, report_dir=None, timestamp=None, services=None, skipped_servi self._load_metadata() - self.services = self.services_config(self.metadata, thread_config) + self.services = self.services_config(self.credentials, thread_config) supported_services = vars(self.services).keys() self.service_list = self._build_services_list(supported_services, services, skipped_services) From 919d92a067852ed3bc3824d47aba75b66cee209c Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 27 Mar 2019 15:44:27 -0400 Subject: [PATCH 623/667] Cleaned main --- ScoutSuite/__main__.py | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/ScoutSuite/__main__.py b/ScoutSuite/__main__.py index 2f1457279..e3b0afd79 100644 --- a/ScoutSuite/__main__.py +++ b/ScoutSuite/__main__.py @@ -34,8 +34,8 @@ async def main(args=None): print_info('Launching Scout') - auth_strategy = get_authentication_strategy(args.get('provider')) if not args.get('fetch_local'): + auth_strategy = get_authentication_strategy(args.get('provider')) credentials = auth_strategy.authenticate(profile=args.get('profile'), user_account=args.get('user_account'), service_account=args.get('service_account'), @@ -51,8 +51,8 @@ async def main(args=None): password=args.get('password') ) - if not credentials: - return 401 + if not credentials: + return 401 # Create a cloud provider object @@ -78,24 +78,6 @@ async def main(args=None): # Complete run, including pulling data from provider if not args.get('fetch_local'): - # # Authenticate to the cloud provider - # authenticated = cloud_provider.authenticate(profile=args.get('profile'), - # user_account=args.get('user_account'), - # service_account=args.get('service_account'), - # cli=args.get('cli'), - # msi=args.get('msi'), - # service_principal=args.get('service_principal'), - # file_auth=args.get('file_auth'), - # tenant_id=args.get('tenant_id'), - # subscription_id=args.get('subscription_id'), - # client_id=args.get('client_id'), - # client_secret=args.get('client_secret'), - # username=args.get('username'), - # password=args.get('password')) - - # if not authenticated: - # return 401 - # Fetch data from provider APIs try: print_info('Gathering data from APIs') From ac65dba1fb4705f8d516b68763dc360810d82b91 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 27 Mar 2019 15:44:37 -0400 Subject: [PATCH 624/667] Injected facade in AWS services --- ScoutSuite/providers/aws/configs/services.py | 44 ++++++++++--------- ScoutSuite/providers/aws/facade/s3.py | 20 ++++----- .../aws/resources/awslambda/service.py | 4 +- .../aws/resources/cloudformation/service.py | 4 +- .../aws/resources/cloudtrail/service.py | 9 ++-- .../aws/resources/cloudwatch/service.py | 6 +-- .../aws/resources/directconnect/service.py | 6 +-- .../providers/aws/resources/ec2/service.py | 9 ++-- .../providers/aws/resources/efs/service.py | 6 +-- .../aws/resources/elasticache/service.py | 14 +++--- .../providers/aws/resources/elb/service.py | 5 ++- .../providers/aws/resources/elbv2/service.py | 5 ++- .../providers/aws/resources/emr/service.py | 15 ++++--- .../providers/aws/resources/iam/service.py | 7 +-- .../providers/aws/resources/rds/instances.py | 1 - .../providers/aws/resources/rds/service.py | 10 +++-- .../aws/resources/rds/subnetgroups.py | 1 - .../aws/resources/redshift/service.py | 7 +-- ScoutSuite/providers/aws/resources/regions.py | 8 +--- .../providers/aws/resources/resources.py | 1 - .../aws/resources/route53/service.py | 5 ++- .../providers/aws/resources/s3/service.py | 8 +--- .../providers/aws/resources/ses/service.py | 5 ++- .../providers/aws/resources/sns/service.py | 5 ++- .../providers/aws/resources/sqs/service.py | 5 ++- .../providers/aws/resources/vpc/service.py | 12 ++--- .../providers/azure/configs/services.py | 2 +- ScoutSuite/providers/gcp/configs/services.py | 2 +- 28 files changed, 116 insertions(+), 110 deletions(-) diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index e1f23db40..94c63cb69 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -21,6 +21,7 @@ from ScoutSuite.providers.aws.resources.ses.service import SES from ScoutSuite.providers.aws.resources.sns.service import SNS from ScoutSuite.providers.base.configs.services import BaseServicesConfig +from ScoutSuite.providers.aws.facade.facade import AWSFacade try: from ScoutSuite.providers.aws.resources.dynamodb.service_private import DynamoDB @@ -54,27 +55,28 @@ class AWSServicesConfig(BaseServicesConfig): def __init__(self, credentials=None, thread_config=4, **kwargs): super(AWSServicesConfig, self).__init__(credentials, thread_config) - super(AWSServicesConfig, self).__init__(metadata, thread_config) - self.cloudwatch = CloudWatch() - self.cloudformation = CloudFormation() - self.cloudtrail = CloudTrail() - self.directconnect = DirectConnect() - self.ec2 = EC2() - self.efs = EFS() - self.elasticache = ElastiCache() - self.iam = IAM() - self.elb = ELB() - self.elbv2 = ELBv2() - self.emr = EMR() - self.awslambda = Lambdas() - self.route53 = Route53() - self.redshift = Redshift() - self.s3 = S3() - self.rds = RDS() - self.vpc = VPC() - self.sqs = SQS() - self.ses = SES() - self.sns = SNS() + facade = AWSFacade(credentials) + + self.cloudwatch = CloudWatch(facade) + self.cloudformation = CloudFormation(facade) + self.cloudtrail = CloudTrail(facade) + self.directconnect = DirectConnect(facade) + self.ec2 = EC2(facade) + self.efs = EFS(facade) + self.elasticache = ElastiCache(facade) + self.iam = IAM(facade) + self.elb = ELB(facade) + self.elbv2 = ELBv2(facade) + self.emr = EMR(facade) + self.awslambda = Lambdas(facade) + self.route53 = Route53(facade) + self.redshift = Redshift(facade) + self.s3 = S3(facade) + self.rds = RDS(facade) + self.vpc = VPC(facade) + self.sqs = SQS(facade) + self.ses = SES(facade) + self.sns = SNS(facade) try: self.dynamodb = DynamoDB() diff --git a/ScoutSuite/providers/aws/facade/s3.py b/ScoutSuite/providers/aws/facade/s3.py index f01411341..4e0728cc4 100644 --- a/ScoutSuite/providers/aws/facade/s3.py +++ b/ScoutSuite/providers/aws/facade/s3.py @@ -12,7 +12,7 @@ class S3Facade(AWSBaseFacade): async def get_buckets(self): - client = AWSFacadeUtils.get_client('s3', None, self.session) + client = AWSFacadeUtils.get_client('s3', self.session) buckets = (await run_concurrently(client.list_buckets))['Buckets'] for bucket in buckets: @@ -29,7 +29,7 @@ async def get_buckets(self): return buckets async def _get_s3_bucket_location(self, bucket_name): - client = AWSFacadeUtils.get_client('s3', None, self.session) + client = AWSFacadeUtils.get_client('s3', self.session) location = client.get_bucket_location(Bucket=bucket_name) region = location['LocationConstraint'] if location['LocationConstraint'] else 'us-east-1' @@ -40,7 +40,7 @@ async def _get_s3_bucket_location(self, bucket_name): return region async def _set_s3_bucket_logging(self, bucket): - client = AWSFacadeUtils.get_client('s3', bucket['region'], self.session) + client = AWSFacadeUtils.get_client('s3', self.session, bucket['region'],) try: logging = await run_concurrently(lambda: client.get_bucket_logging(Bucket=bucket['Name'])) except Exception as e: @@ -54,7 +54,7 @@ async def _set_s3_bucket_logging(self, bucket): # noinspection PyBroadException async def _set_s3_bucket_versioning(self, bucket): - client = AWSFacadeUtils.get_client('s3', bucket['region'], self.session) + client = AWSFacadeUtils.get_client('s3', self.session, bucket['region']) try: versioning = await run_concurrently(lambda: client.get_bucket_versioning(Bucket=bucket['Name'])) bucket['versioning_status_enabled'] = self._status_to_bool(versioning.get('Status')) @@ -66,7 +66,7 @@ async def _set_s3_bucket_versioning(self, bucket): # noinspection PyBroadException async def _set_s3_bucket_webhosting(self, bucket): - client = AWSFacadeUtils.get_client('s3', bucket['region'], self.session) + client = AWSFacadeUtils.get_client('s3', self.session, bucket['region']) try: result = client.get_bucket_website(Bucket=bucket['Name']) bucket['web_hosting_enabled'] = 'IndexDocument' in result @@ -76,7 +76,7 @@ async def _set_s3_bucket_webhosting(self, bucket): async def _set_s3_bucket_default_encryption(self, bucket): bucket_name = bucket['Name'] - client = AWSFacadeUtils.get_client('s3', bucket['region'], self.session) + client = AWSFacadeUtils.get_client('s3', self.session, bucket['region']) try: await run_concurrently(lambda: client.get_bucket_encryption(Bucket=bucket['Name'])) bucket['default_encryption_enabled'] = True @@ -92,7 +92,7 @@ async def _set_s3_bucket_default_encryption(self, bucket): async def _set_s3_acls(self, bucket, key_name=None): bucket_name = bucket['Name'] - client = AWSFacadeUtils.get_client('s3', bucket['region'], self.session) + client = AWSFacadeUtils.get_client('s3', self.session, bucket['region']) try: grantees = {} if key_name: @@ -122,12 +122,12 @@ async def _set_s3_acls(self, bucket, key_name=None): bucket['grantees'] = {} async def _set_s3_bucket_policy(self, bucket): - client = AWSFacadeUtils.get_client('s3', bucket['region'], self.session) + client = AWSFacadeUtils.get_client('s3', self.session, bucket['region']) try: bucket_policy = await run_concurrently(lambda: client.get_bucket_policy(Bucket=bucket['Name'])) bucket['policy'] = json.loads(bucket_policy['Policy']) - except Exception as e: - if not (type(e) == ClientError and e.response['Error']['Code'] == 'NoSuchBucketPolicy'): + except ClientError as e: + if e.response['Error']['Code'] != 'NoSuchBucketPolicy': print_error('Failed to get bucket policy for %s: %s' % (bucket['Name'], e)) def _set_s3_bucket_secure_transport(self, bucket): diff --git a/ScoutSuite/providers/aws/resources/awslambda/service.py b/ScoutSuite/providers/aws/resources/awslambda/service.py index cab77586d..0a80ced7f 100644 --- a/ScoutSuite/providers/aws/resources/awslambda/service.py +++ b/ScoutSuite/providers/aws/resources/awslambda/service.py @@ -20,5 +20,5 @@ class Lambdas(Regions): (Functions, 'functions') ] - def __init__(self): - super(Lambdas, self).__init__('lambda') + def __init__(self, facade: AWSFacade): + super(Lambdas, self).__init__('lambda', facade) diff --git a/ScoutSuite/providers/aws/resources/cloudformation/service.py b/ScoutSuite/providers/aws/resources/cloudformation/service.py index f4c395d47..8eaef8ba5 100644 --- a/ScoutSuite/providers/aws/resources/cloudformation/service.py +++ b/ScoutSuite/providers/aws/resources/cloudformation/service.py @@ -60,5 +60,5 @@ class CloudFormation(Regions): (Stacks, 'stacks') ] - def __init__(self): - super(CloudFormation, self).__init__('cloudformation') + def __init__(self, facade: AWSFacade): + super(CloudFormation, self).__init__('cloudformation', facade) diff --git a/ScoutSuite/providers/aws/resources/cloudtrail/service.py b/ScoutSuite/providers/aws/resources/cloudtrail/service.py index a2a962c12..1e2c554d3 100644 --- a/ScoutSuite/providers/aws/resources/cloudtrail/service.py +++ b/ScoutSuite/providers/aws/resources/cloudtrail/service.py @@ -1,9 +1,10 @@ +import time + +from ScoutSuite.providers.aws.facade.facade import AWSFacade from ScoutSuite.providers.aws.resources.regions import Regions from ScoutSuite.providers.aws.resources.resources import AWSResources from ScoutSuite.providers.utils import get_non_provider_id -import time - class Trails(AWSResources): async def fetch_all(self, **kwargs): @@ -66,5 +67,5 @@ class CloudTrail(Regions): (Trails, 'trails') ] - def __init__(self): - super(CloudTrail, self).__init__('cloudtrail') + def __init__(self, facade: AWSFacade): + super(CloudTrail, self).__init__('cloudtrail', facade) diff --git a/ScoutSuite/providers/aws/resources/cloudwatch/service.py b/ScoutSuite/providers/aws/resources/cloudwatch/service.py index 29454e18f..2a8db4d1b 100644 --- a/ScoutSuite/providers/aws/resources/cloudwatch/service.py +++ b/ScoutSuite/providers/aws/resources/cloudwatch/service.py @@ -1,8 +1,8 @@ -from ScoutSuite.providers.aws.facade.facade import AWSFacade from ScoutSuite.providers.aws.resources.regions import Regions from ScoutSuite.providers.aws.resources.resources import AWSResources from ScoutSuite.providers.utils import get_non_provider_id +from ScoutSuite.providers.aws.facade.facade import AWSFacade class Alarms(AWSResources): @@ -30,5 +30,5 @@ class CloudWatch(Regions): (Alarms, 'alarms') ] - def __init__(self): - super(CloudWatch, self).__init__('cloudwatch') + def __init__(self, facade: AWSFacade): + super(CloudWatch, self).__init__('cloudwatch', facade) diff --git a/ScoutSuite/providers/aws/resources/directconnect/service.py b/ScoutSuite/providers/aws/resources/directconnect/service.py index 97293a515..032c158ae 100644 --- a/ScoutSuite/providers/aws/resources/directconnect/service.py +++ b/ScoutSuite/providers/aws/resources/directconnect/service.py @@ -5,7 +5,7 @@ class Connections(AWSResources): async def fetch_all(self, **kwargs): - raw_connections = await self.facade.directconnect.get_connections(self.scope['region']) + raw_connections = await self.facade.directconnect.get_connections(self.scope['region']) for raw_connection in raw_connections: name, resource = self._parse_function(raw_connection) self[name] = resource @@ -21,5 +21,5 @@ class DirectConnect(Regions): (Connections, 'connections') ] - def __init__(self): - super(DirectConnect, self).__init__('directconnect') + def __init__(self, facade: AWSFacade): + super(DirectConnect, self).__init__('directconnect', facade) diff --git a/ScoutSuite/providers/aws/resources/ec2/service.py b/ScoutSuite/providers/aws/resources/ec2/service.py index 3258f298d..35b798d17 100644 --- a/ScoutSuite/providers/aws/resources/ec2/service.py +++ b/ScoutSuite/providers/aws/resources/ec2/service.py @@ -1,8 +1,9 @@ -from ScoutSuite.providers.aws.resources.regions import Regions +from ScoutSuite.providers.aws.facade.facade import AWSFacade from ScoutSuite.providers.aws.resources.ec2.ami import AmazonMachineImages -from ScoutSuite.providers.aws.resources.ec2.vpcs import Ec2Vpcs from ScoutSuite.providers.aws.resources.ec2.snapshots import Snapshots from ScoutSuite.providers.aws.resources.ec2.volumes import Volumes +from ScoutSuite.providers.aws.resources.ec2.vpcs import Ec2Vpcs +from ScoutSuite.providers.aws.resources.regions import Regions class EC2(Regions): @@ -13,8 +14,8 @@ class EC2(Regions): (Volumes, 'volumes') ] - def __init__(self): - super(EC2, self).__init__('ec2') + def __init__(self, facade): + super(EC2, self).__init__('ec2', facade) async def fetch_all(self, credentials=None, regions=None, partition_name='aws'): await super(EC2, self).fetch_all(credentials, regions, partition_name) diff --git a/ScoutSuite/providers/aws/resources/efs/service.py b/ScoutSuite/providers/aws/resources/efs/service.py index 42f4092f6..a150cf32b 100644 --- a/ScoutSuite/providers/aws/resources/efs/service.py +++ b/ScoutSuite/providers/aws/resources/efs/service.py @@ -1,6 +1,6 @@ +from ScoutSuite.providers.aws.facade.facade import AWSFacade from ScoutSuite.providers.aws.resources.regions import Regions from ScoutSuite.providers.aws.resources.resources import AWSResources -from ScoutSuite.providers.aws.facade.facade import AWSFacade class FileSystems(AWSResources): @@ -24,5 +24,5 @@ class EFS(Regions): (FileSystems, 'filesystems') ] - def __init__(self): - super(EFS, self).__init__('efs') + def __init__(self, facade: AWSFacade): + super(EFS, self).__init__('efs', facade) diff --git a/ScoutSuite/providers/aws/resources/elasticache/service.py b/ScoutSuite/providers/aws/resources/elasticache/service.py index 6e5dfee9f..36f52a04c 100644 --- a/ScoutSuite/providers/aws/resources/elasticache/service.py +++ b/ScoutSuite/providers/aws/resources/elasticache/service.py @@ -1,8 +1,10 @@ from ScoutSuite.providers.aws.facade.facade import AWSFacade -from ScoutSuite.providers.aws.resources.regions import Regions +from ScoutSuite.providers.aws.resources.elasticache.parametergroups import \ + ParameterGroups +from ScoutSuite.providers.aws.resources.elasticache.securitygroups import \ + SecurityGroups from ScoutSuite.providers.aws.resources.elasticache.vpcs import ElastiCacheVpcs -from ScoutSuite.providers.aws.resources.elasticache.securitygroups import SecurityGroups -from ScoutSuite.providers.aws.resources.elasticache.parametergroups import ParameterGroups +from ScoutSuite.providers.aws.resources.regions import Regions class ElastiCache(Regions): @@ -12,8 +14,8 @@ class ElastiCache(Regions): (ParameterGroups, 'parameter_groups') ] - def __init__(self): - super(ElastiCache, self).__init__('elasticache') + def __init__(self, facade: AWSFacade): + super(ElastiCache, self).__init__('elasticache', facade) async def fetch_all(self, credentials=None, regions=None, partition_name='aws'): await super(ElastiCache, self).fetch_all(credentials, regions, partition_name) @@ -26,4 +28,4 @@ async def fetch_all(self, credentials=None, regions=None, partition_name='aws'): # We do not want the parameter groups to be part of the resources count, as it is usually in # the three of four digits and would make the resources count confusing. - self.pop('parameter_groups_count') \ No newline at end of file + self.pop('parameter_groups_count') diff --git a/ScoutSuite/providers/aws/resources/elb/service.py b/ScoutSuite/providers/aws/resources/elb/service.py index dd61c31fd..f35f2fac5 100644 --- a/ScoutSuite/providers/aws/resources/elb/service.py +++ b/ScoutSuite/providers/aws/resources/elb/service.py @@ -1,3 +1,4 @@ +from ScoutSuite.providers.aws.facade.facade import AWSFacade from ScoutSuite.providers.aws.resources.regions import Regions from .vpcs import ELBVpcs @@ -8,5 +9,5 @@ class ELB(Regions): (ELBVpcs, 'vpcs') ] - def __init__(self): - super(ELB, self).__init__('elb') + def __init__(self, facade: AWSFacade): + super(ELB, self).__init__('elb', facade) diff --git a/ScoutSuite/providers/aws/resources/elbv2/service.py b/ScoutSuite/providers/aws/resources/elbv2/service.py index 6be76ee48..818a7a80b 100644 --- a/ScoutSuite/providers/aws/resources/elbv2/service.py +++ b/ScoutSuite/providers/aws/resources/elbv2/service.py @@ -1,3 +1,4 @@ +from ScoutSuite.providers.aws.facade.facade import AWSFacade from ScoutSuite.providers.aws.resources.regions import Regions from .vpcs import ELBv2Vpcs @@ -8,5 +9,5 @@ class ELBv2(Regions): (ELBv2Vpcs, 'vpcs') ] - def __init__(self): - super(ELBv2, self).__init__('elbv2') + def __init__(self, facade: AWSFacade): + super(ELBv2, self).__init__('elbv2', facade) diff --git a/ScoutSuite/providers/aws/resources/emr/service.py b/ScoutSuite/providers/aws/resources/emr/service.py index 084245cc0..61c9622db 100644 --- a/ScoutSuite/providers/aws/resources/emr/service.py +++ b/ScoutSuite/providers/aws/resources/emr/service.py @@ -1,6 +1,7 @@ from ScoutSuite.providers.aws.facade.facade import AWSFacade from ScoutSuite.providers.aws.resources.regions import Regions -from ScoutSuite.providers.aws.resources.resources import AWSResources, AWSCompositeResources +from ScoutSuite.providers.aws.resources.resources import (AWSCompositeResources, + AWSResources) class EMRClusters(AWSResources): @@ -34,13 +35,15 @@ class EMR(Regions): (EMRVpcs, 'vpcs') ] - def __init__(self): - super(EMR, self).__init__('emr') + def __init__(self, facade: AWSFacade): + super(EMR, self).__init__('emr', facade) async def fetch_all(self, credentials=None, regions=None, partition_name='aws'): await super(EMR, self).fetch_all(credentials, regions, partition_name) for region in self['regions']: - self['regions'][region]['clusters_count'] = sum([len(vpc['clusters']) for vpc in self['regions'][region]['vpcs'].values()]) - - self['clusters_count'] = sum([region['clusters_count'] for region in self['regions'].values()]) + self['regions'][region]['clusters_count'] = sum( + [len(vpc['clusters']) for vpc in self['regions'][region]['vpcs'].values()]) + + self['clusters_count'] = sum( + [region['clusters_count'] for region in self['regions'].values()]) diff --git a/ScoutSuite/providers/aws/resources/iam/service.py b/ScoutSuite/providers/aws/resources/iam/service.py index 7b92edadb..b68de7770 100644 --- a/ScoutSuite/providers/aws/resources/iam/service.py +++ b/ScoutSuite/providers/aws/resources/iam/service.py @@ -20,14 +20,11 @@ class IAM(AWSCompositeResources): (PasswordPolicy, 'password_policy') ] - def __init__(self): - # TODO: Should be injected - self.facade = AWSFacade() + def __init__(self, facade: AWSFacade): + self.facade = facade self.service = 'iam' async def fetch_all(self, credentials, regions=None, partition_name='aws'): - # TODO: This should not be set here, the facade should be injected and already authenticated - self.facade._set_session(credentials) await self._fetch_children(self, {}) # We do not want the report to count the password policies as resources, they aren't really resources. diff --git a/ScoutSuite/providers/aws/resources/rds/instances.py b/ScoutSuite/providers/aws/resources/rds/instances.py index 9a13350ce..821db9b89 100644 --- a/ScoutSuite/providers/aws/resources/rds/instances.py +++ b/ScoutSuite/providers/aws/resources/rds/instances.py @@ -1,4 +1,3 @@ -from ScoutSuite.providers.aws.facade.facade import AWSFacade from ScoutSuite.providers.aws.resources.regions import Regions from ScoutSuite.providers.aws.resources.resources import AWSResources diff --git a/ScoutSuite/providers/aws/resources/rds/service.py b/ScoutSuite/providers/aws/resources/rds/service.py index 8b6d8bf3a..bc8827a1d 100644 --- a/ScoutSuite/providers/aws/resources/rds/service.py +++ b/ScoutSuite/providers/aws/resources/rds/service.py @@ -1,7 +1,9 @@ -from ScoutSuite.providers.aws.resources.regions import Regions -from ScoutSuite.providers.aws.resources.rds.vpcs import RDSVpcs +from ScoutSuite.providers.aws.facade.facade import AWSFacade from ScoutSuite.providers.aws.resources.rds.parametergroups import ParameterGroups from ScoutSuite.providers.aws.resources.rds.securitygroups import SecurityGroups +from ScoutSuite.providers.aws.resources.rds.vpcs import RDSVpcs +from ScoutSuite.providers.aws.resources.regions import Regions + class RDS(Regions): _children = [ @@ -10,8 +12,8 @@ class RDS(Regions): (SecurityGroups, 'security_groups') ] - def __init__(self): - super(RDS, self).__init__('rds') + def __init__(self, facade: AWSFacade): + super(RDS, self).__init__('rds', facade) async def fetch_all(self, credentials=None, regions=None, partition_name='aws'): await super(RDS, self).fetch_all(credentials, regions, partition_name) diff --git a/ScoutSuite/providers/aws/resources/rds/subnetgroups.py b/ScoutSuite/providers/aws/resources/rds/subnetgroups.py index af4dc788b..68999fd7d 100644 --- a/ScoutSuite/providers/aws/resources/rds/subnetgroups.py +++ b/ScoutSuite/providers/aws/resources/rds/subnetgroups.py @@ -1,4 +1,3 @@ -from ScoutSuite.providers.aws.facade.facade import AWSFacade from ScoutSuite.providers.aws.resources.regions import Regions from ScoutSuite.providers.aws.resources.resources import AWSResources diff --git a/ScoutSuite/providers/aws/resources/redshift/service.py b/ScoutSuite/providers/aws/resources/redshift/service.py index d8b24ae4c..8a5f4ee1d 100644 --- a/ScoutSuite/providers/aws/resources/redshift/service.py +++ b/ScoutSuite/providers/aws/resources/redshift/service.py @@ -1,8 +1,9 @@ +from ScoutSuite.providers.aws.facade.facade import AWSFacade from ScoutSuite.providers.aws.resources.regions import Regions -from .vpcs import RedshiftVpcs from .cluster_parameter_groups import ClusterParameterGroups from .cluster_security_groups import ClusterSecurityGroups +from .vpcs import RedshiftVpcs class Redshift(Regions): @@ -12,5 +13,5 @@ class Redshift(Regions): (ClusterSecurityGroups, 'security_groups') ] - def __init__(self): - super(Redshift, self).__init__('redshift') + def __init__(self, facade: AWSFacade): + super(Redshift, self).__init__('redshift', facade) diff --git a/ScoutSuite/providers/aws/resources/regions.py b/ScoutSuite/providers/aws/resources/regions.py index 484a1404b..8c7616503 100644 --- a/ScoutSuite/providers/aws/resources/regions.py +++ b/ScoutSuite/providers/aws/resources/regions.py @@ -7,15 +7,11 @@ class Regions(AWSCompositeResources, metaclass=abc.ABCMeta): - def __init__(self, service): + def __init__(self, service, facade): self.service = service - # TODO: Should be injected - self.facade = AWSFacade() + self.facade = facade async def fetch_all(self, credentials, regions=None, partition_name='aws'): - # TODO: This should not be set here, the facade should be injected and already authenticated - self.facade._set_session(credentials) - self['regions'] = {} account_id = get_aws_account_id(credentials) for region in await self.facade.build_region_list(self.service, regions, partition_name): diff --git a/ScoutSuite/providers/aws/resources/resources.py b/ScoutSuite/providers/aws/resources/resources.py index ca87905ec..3a2180123 100644 --- a/ScoutSuite/providers/aws/resources/resources.py +++ b/ScoutSuite/providers/aws/resources/resources.py @@ -4,7 +4,6 @@ import asyncio from ScoutSuite.providers.base.configs.resources import Resources, CompositeResources -from ScoutSuite.providers.aws.facade.facade import AWSFacade class AWSResources(Resources, metaclass=abc.ABCMeta): diff --git a/ScoutSuite/providers/aws/resources/route53/service.py b/ScoutSuite/providers/aws/resources/route53/service.py index a87a687da..3858aa8c8 100644 --- a/ScoutSuite/providers/aws/resources/route53/service.py +++ b/ScoutSuite/providers/aws/resources/route53/service.py @@ -1,3 +1,4 @@ +from ScoutSuite.providers.aws.facade.facade import AWSFacade from ScoutSuite.providers.aws.resources.regions import Regions from ScoutSuite.providers.aws.resources.resources import AWSResources from ScoutSuite.providers.utils import get_non_provider_id @@ -21,5 +22,5 @@ class Route53(Regions): (Route53Domains, 'domains') ] - def __init__(self): - super(Route53, self).__init__('route53domains') + def __init__(self, facade: AWSFacade): + super(Route53, self).__init__('route53domains', facade) diff --git a/ScoutSuite/providers/aws/resources/s3/service.py b/ScoutSuite/providers/aws/resources/s3/service.py index bba9e1081..a77dafe31 100644 --- a/ScoutSuite/providers/aws/resources/s3/service.py +++ b/ScoutSuite/providers/aws/resources/s3/service.py @@ -41,12 +41,8 @@ class S3(AWSCompositeResources): (Buckets, 'buckets') ] - def __init__(self): - super(S3, self).__init__('s3', {}) - # TODO: Should be injected - self.facade = AWSFacade() + def __init__(self, facade: AWSFacade): + super(S3, self,).__init__(facade, {}) async def fetch_all(self, credentials, regions=None, partition_name='aws'): - # TODO: This should not be set here, the facade should be injected and already authenticated - self.facade._set_session(credentials) await self._fetch_children(self, {}) diff --git a/ScoutSuite/providers/aws/resources/ses/service.py b/ScoutSuite/providers/aws/resources/ses/service.py index 6cb29e037..0a873ada1 100644 --- a/ScoutSuite/providers/aws/resources/ses/service.py +++ b/ScoutSuite/providers/aws/resources/ses/service.py @@ -1,3 +1,4 @@ +from ScoutSuite.providers.aws.facade.facade import AWSFacade from ScoutSuite.providers.aws.resources.regions import Regions from .identities import Identities @@ -8,5 +9,5 @@ class SES(Regions): (Identities, 'identities') ] - def __init__(self): - super(SES, self).__init__('ses') + def __init__(self, facade: AWSFacade): + super(SES, self).__init__('ses', facade) diff --git a/ScoutSuite/providers/aws/resources/sns/service.py b/ScoutSuite/providers/aws/resources/sns/service.py index 66d1b38af..98afd1352 100644 --- a/ScoutSuite/providers/aws/resources/sns/service.py +++ b/ScoutSuite/providers/aws/resources/sns/service.py @@ -1,3 +1,4 @@ +from ScoutSuite.providers.aws.facade.facade import AWSFacade from ScoutSuite.providers.aws.resources.regions import Regions from .topics import Topics @@ -8,5 +9,5 @@ class SNS(Regions): (Topics, 'topics') ] - def __init__(self): - super(SNS, self).__init__('sns') + def __init__(self, facade: AWSFacade): + super(SNS, self).__init__('sns', facade) diff --git a/ScoutSuite/providers/aws/resources/sqs/service.py b/ScoutSuite/providers/aws/resources/sqs/service.py index 254786aaf..c8bdfe083 100644 --- a/ScoutSuite/providers/aws/resources/sqs/service.py +++ b/ScoutSuite/providers/aws/resources/sqs/service.py @@ -1,5 +1,6 @@ import json +from ScoutSuite.providers.aws.facade.facade import AWSFacade from ScoutSuite.providers.aws.resources.regions import Regions from ScoutSuite.providers.aws.resources.resources import AWSResources @@ -33,5 +34,5 @@ class SQS(Regions): (RegionalQueues, 'queues') ] - def __init__(self): - super(SQS, self).__init__('sqs') + def __init__(self, facade: AWSFacade): + super(SQS, self).__init__('sqs', facade) diff --git a/ScoutSuite/providers/aws/resources/vpc/service.py b/ScoutSuite/providers/aws/resources/vpc/service.py index 5853861c1..f3875375a 100644 --- a/ScoutSuite/providers/aws/resources/vpc/service.py +++ b/ScoutSuite/providers/aws/resources/vpc/service.py @@ -1,10 +1,11 @@ import netaddr -from ScoutSuite.providers.aws.resources.regions import Regions from ScoutSuite.core.fs import read_ip_ranges +from ScoutSuite.providers.aws.facade.facade import AWSFacade +from ScoutSuite.providers.aws.resources.regions import Regions -from .vpcs import RegionalVpcs from .flow_logs import FlowLogs +from .vpcs import RegionalVpcs known_cidrs = {'0.0.0.0/0': 'All'} aws_ip_ranges = {} @@ -16,9 +17,9 @@ class VPC(Regions): (FlowLogs, 'flow_logs'), ] - def __init__(self): + def __init__(self, facade: AWSFacade): # VPC is not a real service but a subset of ec2: - super(VPC, self).__init__('ec2') + super(VPC, self).__init__('ec2', facade) # TODO: move these helpers elsewhere: @@ -35,7 +36,8 @@ def put_cidr_name(aws_config, current_config, path, current_path, resource_id, c if cidr in known_cidrs: cidr_name = known_cidrs[cidr] else: - cidr_name = get_cidr_name(cidr, callback_args['ip_ranges'], callback_args['ip_ranges_name_key']) + cidr_name = get_cidr_name( + cidr, callback_args['ip_ranges'], callback_args['ip_ranges_name_key']) known_cidrs[cidr] = cidr_name cidr_list.append({'CIDR': cidr, 'CIDRName': cidr_name}) current_config['cidrs'] = cidr_list diff --git a/ScoutSuite/providers/azure/configs/services.py b/ScoutSuite/providers/azure/configs/services.py index ecaed2f31..8fb305298 100644 --- a/ScoutSuite/providers/azure/configs/services.py +++ b/ScoutSuite/providers/azure/configs/services.py @@ -26,7 +26,7 @@ class AzureServicesConfig(BaseServicesConfig): - def __init__(self, metadata=None, thread_config=4, **kwargs): + def __init__(self, credentials=None, thread_config=4, **kwargs): self.storageaccounts = StorageAccounts() self.securitycenter = SecurityCenter() diff --git a/ScoutSuite/providers/gcp/configs/services.py b/ScoutSuite/providers/gcp/configs/services.py index 0693601a4..0b99426b7 100644 --- a/ScoutSuite/providers/gcp/configs/services.py +++ b/ScoutSuite/providers/gcp/configs/services.py @@ -16,7 +16,7 @@ class GCPServicesConfig(BaseServicesConfig): - def __init__(self, metadata=None, thread_config=4, projects=None, **kwargs): + def __init__(self, credentials=None, thread_config=4, projects=None, **kwargs): projects = [] if projects is None else projects From a0fb9dcdaff4b1d8e06cd39717183255fbbc23b2 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 27 Mar 2019 17:55:57 -0400 Subject: [PATCH 625/667] Created main azure facade --- ScoutSuite/providers/azure/facade/facade.py | 23 +++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 ScoutSuite/providers/azure/facade/facade.py diff --git a/ScoutSuite/providers/azure/facade/facade.py b/ScoutSuite/providers/azure/facade/facade.py new file mode 100644 index 000000000..2ca639e24 --- /dev/null +++ b/ScoutSuite/providers/azure/facade/facade.py @@ -0,0 +1,23 @@ +from ScoutSuite.providers.azure.facade.appgateway_private import AppGatewayFacade +from ScoutSuite.providers.azure.facade.appservice_private import AppServiceFacade +from ScoutSuite.providers.azure.facade.keyvault import KeyVaultFacade +from ScoutSuite.providers.azure.facade.loadbalancer_private import LoadBalancerFacade +from ScoutSuite.providers.azure.facade.network import NetworkFacade +from ScoutSuite.providers.azure.facade.rediscache_private import RedisCacheFacade +from ScoutSuite.providers.azure.facade.securitycenter import SecurityCenterFacade +from ScoutSuite.providers.azure.facade.sqldatabase import SQLDatabaseFacade +from ScoutSuite.providers.azure.facade.storageaccounts import StorageAccountsFacade +from ScoutSuite.providers.azure.authentication_strategy import AzureCredentials + + +class AzureFacade(): + def __init__(self, credentials: AzureCredentials): + self.appgateway = AppGatewayFacade(credentials.credentials, credentials.subscription_id) + self.appservice = AppServiceFacade(credentials.credentials, credentials.subscription_id) + self.keyvault = KeyVaultFacade(credentials.credentials, credentials.subscription_id) + self.loadbalancer = LoadBalancerFacade(credentials.credentials, credentials.subscription_id) + self.network = NetworkFacade(credentials.credentials, credentials.subscription_id) + self.rediscache = RedisCacheFacade(credentials.credentials, credentials.subscription_id) + self.securitycenter = SecurityCenterFacade(credentials.credentials, credentials.subscription_id) + self.sqldatabase = SQLDatabaseFacade(credentials.credentials, credentials.subscription_id) + self.storageaccounts = StorageAccountsFacade(credentials.credentials, credentials.subscription_id) \ No newline at end of file From bca9069f0cabf8d70486bda579b8123a446f6519 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 27 Mar 2019 17:56:30 -0400 Subject: [PATCH 626/667] Injected facade, typed facade --- ScoutSuite/providers/aws/configs/services.py | 6 ++-- ScoutSuite/providers/aws/resources/regions.py | 2 +- .../providers/azure/configs/services.py | 30 +++++++++++-------- .../azure/resources/keyvault/key_vaults.py | 10 +++---- .../network/network_security_groups.py | 5 ++-- .../resources/network/network_watchers.py | 5 ++-- .../azure/resources/network/networks.py | 6 +--- .../providers/azure/resources/resources.py | 4 +++ .../auto_provisioning_settings.py | 8 +++-- .../resources/securitycenter/pricings.py | 5 ++-- .../securitycenter/security_center.py | 9 ++---- .../securitycenter/security_contacts.py | 5 ++-- .../database_blob_auditing_policies.py | 5 ++-- .../database_threat_detection_policies.py | 5 ++-- .../azure/resources/sqldatabase/databases.py | 5 ++-- .../sqldatabase/replication_links.py | 5 ++-- .../server_azure_ad_administrators.py | 5 ++-- .../server_blob_auditing_policies.py | 2 +- .../server_security_alert_policies.py | 2 +- .../azure/resources/sqldatabase/servers.py | 7 ++--- .../transparent_data_encryptions.py | 2 +- .../storageaccounts/blob_containers.py | 5 ++-- .../storageaccounts/storageaccounts.py | 8 ++--- ScoutSuite/providers/gcp/configs/services.py | 2 +- 24 files changed, 77 insertions(+), 71 deletions(-) diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index 94c63cb69..9a7145230 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -79,9 +79,9 @@ def __init__(self, credentials=None, thread_config=4, **kwargs): self.sns = SNS(facade) try: - self.dynamodb = DynamoDB() - self.config = Config() - self.kms = KMS() + self.dynamodb = DynamoDB(facade) + self.config = Config(facade) + self.kms = KMS(facade) except (NameError, TypeError): pass diff --git a/ScoutSuite/providers/aws/resources/regions.py b/ScoutSuite/providers/aws/resources/regions.py index 8c7616503..24cfe6b1c 100644 --- a/ScoutSuite/providers/aws/resources/regions.py +++ b/ScoutSuite/providers/aws/resources/regions.py @@ -7,7 +7,7 @@ class Regions(AWSCompositeResources, metaclass=abc.ABCMeta): - def __init__(self, service, facade): + def __init__(self, service: str, facade: AWSFacade): self.service = service self.facade = facade diff --git a/ScoutSuite/providers/azure/configs/services.py b/ScoutSuite/providers/azure/configs/services.py index 8fb305298..02fccae6b 100644 --- a/ScoutSuite/providers/azure/configs/services.py +++ b/ScoutSuite/providers/azure/configs/services.py @@ -1,11 +1,14 @@ # -*- coding: utf-8 -*- -from ScoutSuite.providers.base.configs.services import BaseServicesConfig +from ScoutSuite.providers.azure.authentication_strategy import AzureCredentials +from ScoutSuite.providers.azure.facade.facade import AzureFacade +from ScoutSuite.providers.azure.resources.keyvault.key_vaults import KeyVaults +from ScoutSuite.providers.azure.resources.network.networks import Networks from ScoutSuite.providers.azure.resources.securitycenter.security_center import SecurityCenter from ScoutSuite.providers.azure.resources.sqldatabase.servers import Servers from ScoutSuite.providers.azure.resources.storageaccounts.storageaccounts import StorageAccounts -from ScoutSuite.providers.azure.resources.network.networks import Networks -from ScoutSuite.providers.azure.resources.keyvault.key_vaults import KeyVaults +from ScoutSuite.providers.base.configs.services import BaseServicesConfig + try: from ScoutSuite.providers.azure.resources.appgateway_private.application_gateways_private import ApplicationGateways except ImportError: @@ -26,28 +29,29 @@ class AzureServicesConfig(BaseServicesConfig): - def __init__(self, credentials=None, thread_config=4, **kwargs): + def __init__(self, credentials: AzureCredentials = None, thread_config=4, **kwargs): + facade = AzureFacade(credentials) - self.storageaccounts = StorageAccounts() - self.securitycenter = SecurityCenter() - self.sqldatabase = Servers() - self.network = Networks() - self.keyvault = KeyVaults() + self.storageaccounts = StorageAccounts(facade) + self.securitycenter = SecurityCenter(facade) + self.sqldatabase = Servers(facade) + self.network = Networks(facade) + self.keyvault = KeyVaults(facade) try: - self.appgateway = ApplicationGateways() + self.appgateway = ApplicationGateways(facade) except NameError: pass try: - self.rediscache = RedisCaches() + self.rediscache = RedisCaches(facade) except NameError: pass try: - self.appservice = WebApplications() + self.appservice = WebApplications(facade) except NameError: pass try: - self.loadbalancer = LoadBalancers() + self.loadbalancer = LoadBalancers(facade) except NameError: pass diff --git a/ScoutSuite/providers/azure/resources/keyvault/key_vaults.py b/ScoutSuite/providers/azure/resources/keyvault/key_vaults.py index 3c6d3d6e5..1cb3fd0a7 100644 --- a/ScoutSuite/providers/azure/resources/keyvault/key_vaults.py +++ b/ScoutSuite/providers/azure/resources/keyvault/key_vaults.py @@ -1,15 +1,15 @@ +from ScoutSuite.providers.azure.facade.facade import AzureFacade from ScoutSuite.providers.base.configs.resources import Resources -from ScoutSuite.providers.azure.facade.keyvault import KeyVaultFacade from ScoutSuite.providers.utils import get_non_provider_id class KeyVaults(Resources): - async def fetch_all(self, credentials, **kwargs): - # TODO: build that facade somewhere else: - facade = KeyVaultFacade(credentials.credentials, credentials.subscription_id) + def __init__(self, facade: AzureFacade): + self.facade = facade + async def fetch_all(self, credentials, **kwargs): self['vaults'] = {} - for raw_vault in await facade.get_key_vaults(): + for raw_vault in await self.facade.keyvault.get_key_vaults(): id, vault = self._parse_key_vault(raw_vault) self['vaults'][id] = vault diff --git a/ScoutSuite/providers/azure/resources/network/network_security_groups.py b/ScoutSuite/providers/azure/resources/network/network_security_groups.py index 6e65b65a0..44afff4b9 100644 --- a/ScoutSuite/providers/azure/resources/network/network_security_groups.py +++ b/ScoutSuite/providers/azure/resources/network/network_security_groups.py @@ -1,12 +1,13 @@ +from ScoutSuite.providers.azure.facade.facade import AzureFacade from ScoutSuite.providers.base.configs.resources import Resources class NetworkSecurityGroups(Resources): - def __init__(self, facade): + def __init__(self, facade: AzureFacade): self.facade = facade async def fetch_all(self): - for raw_group in await self.facade.get_network_security_groups(): + for raw_group in await self.facade.network.get_network_security_groups(): id, network_security_group = self._parse_network_security_group(raw_group) self[id] = network_security_group diff --git a/ScoutSuite/providers/azure/resources/network/network_watchers.py b/ScoutSuite/providers/azure/resources/network/network_watchers.py index 0121f2f1b..27fd3746a 100644 --- a/ScoutSuite/providers/azure/resources/network/network_watchers.py +++ b/ScoutSuite/providers/azure/resources/network/network_watchers.py @@ -1,12 +1,13 @@ +from ScoutSuite.providers.azure.facade.facade import AzureFacade from ScoutSuite.providers.base.configs.resources import Resources class NetworkWatchers(Resources): - def __init__(self, facade): + def __init__(self, facade: AzureFacade): self.facade = facade async def fetch_all(self): - for raw_watcher in await self.facade.get_network_watchers(): + for raw_watcher in await self.facade.network.get_network_watchers(): id, network_watcher = self._parse_network_watcher(raw_watcher) self[id] = network_watcher diff --git a/ScoutSuite/providers/azure/resources/network/networks.py b/ScoutSuite/providers/azure/resources/network/networks.py index e872d5b44..03ffe9f26 100644 --- a/ScoutSuite/providers/azure/resources/network/networks.py +++ b/ScoutSuite/providers/azure/resources/network/networks.py @@ -1,5 +1,4 @@ from ScoutSuite.providers.azure.resources.resources import AzureCompositeResources -from ScoutSuite.providers.azure.facade.network import NetworkFacade from .network_security_groups import NetworkSecurityGroups from .network_watchers import NetworkWatchers @@ -12,10 +11,7 @@ class Networks(AzureCompositeResources): ] async def fetch_all(self, credentials, **kwargs): - # TODO: build that facade somewhere else: - facade = NetworkFacade(credentials.credentials, credentials.subscription_id) - - await self._fetch_children(parent=self, facade=facade) + await self._fetch_children(parent=self, facade=self.facade) self['network_security_groups_count'] = len(self['network_security_groups']) self['network_watchers_count'] = len(self['network_watchers']) diff --git a/ScoutSuite/providers/azure/resources/resources.py b/ScoutSuite/providers/azure/resources/resources.py index 5523bd4e7..5f73a6022 100644 --- a/ScoutSuite/providers/azure/resources/resources.py +++ b/ScoutSuite/providers/azure/resources/resources.py @@ -2,11 +2,15 @@ import asyncio from ScoutSuite.providers.base.configs.resources import CompositeResources +from ScoutSuite.providers.azure.facade.facade import AzureFacade # TODO: add docstrings. class AzureCompositeResources(CompositeResources, metaclass=abc.ABCMeta): + def __init__(self, facade: AzureFacade): + self.facade = facade + async def _fetch_children(self, parent, **kwargs): children = [(child_class(**kwargs), child_name) for (child_class, child_name) in self._children] # fetch all children concurrently: diff --git a/ScoutSuite/providers/azure/resources/securitycenter/auto_provisioning_settings.py b/ScoutSuite/providers/azure/resources/securitycenter/auto_provisioning_settings.py index bacb472c0..283eec431 100644 --- a/ScoutSuite/providers/azure/resources/securitycenter/auto_provisioning_settings.py +++ b/ScoutSuite/providers/azure/resources/securitycenter/auto_provisioning_settings.py @@ -1,13 +1,15 @@ +from ScoutSuite.providers.azure.facade.facade import AzureFacade from ScoutSuite.providers.base.configs.resources import Resources class AutoProvisioningSettings(Resources): - def __init__(self, facade): + def __init__(self, facade: AzureFacade): self.facade = facade async def fetch_all(self): - for raw_settings in await self.facade.get_auto_provisioning_settings(): - id, auto_provisioning_settings = self._parse_auto_provisioning_settings(raw_settings) + for raw_settings in await self.facade.securitycenter.get_auto_provisioning_settings(): + id, auto_provisioning_settings = self._parse_auto_provisioning_settings( + raw_settings) self[id] = auto_provisioning_settings def _parse_auto_provisioning_settings(self, auto_provisioning_settings): diff --git a/ScoutSuite/providers/azure/resources/securitycenter/pricings.py b/ScoutSuite/providers/azure/resources/securitycenter/pricings.py index 48fc7253e..9eb370fb3 100644 --- a/ScoutSuite/providers/azure/resources/securitycenter/pricings.py +++ b/ScoutSuite/providers/azure/resources/securitycenter/pricings.py @@ -1,13 +1,14 @@ +from ScoutSuite.providers.azure.facade.facade import AzureFacade from ScoutSuite.providers.base.configs.resources import Resources class Pricings(Resources): - def __init__(self, facade): + def __init__(self, facade: AzureFacade): self.facade = facade async def fetch_all(self): - for raw_pricing in await self.facade.get_pricings(): + for raw_pricing in await self.facade.securitycenter.get_pricings(): id, pricing = self._parse_pricing(raw_pricing) self[id] = pricing diff --git a/ScoutSuite/providers/azure/resources/securitycenter/security_center.py b/ScoutSuite/providers/azure/resources/securitycenter/security_center.py index 6b9109f2e..5e3463951 100644 --- a/ScoutSuite/providers/azure/resources/securitycenter/security_center.py +++ b/ScoutSuite/providers/azure/resources/securitycenter/security_center.py @@ -1,5 +1,4 @@ from ScoutSuite.providers.azure.resources.resources import AzureCompositeResources -from ScoutSuite.providers.azure.facade.securitycenter import SecurityCenterFacade from .auto_provisioning_settings import AutoProvisioningSettings from .pricings import Pricings @@ -14,11 +13,9 @@ class SecurityCenter(AzureCompositeResources): ] async def fetch_all(self, credentials, **kwargs): - # TODO: build that facade somewhere else: - facade = SecurityCenterFacade(credentials.credentials, credentials.subscription_id) + await self._fetch_children(parent=self, facade=self.facade) - await self._fetch_children(parent=self, facade=facade) - - self['auto_provisioning_settings_count'] = len(self['auto_provisioning_settings']) + self['auto_provisioning_settings_count'] = len( + self['auto_provisioning_settings']) self['pricings_count'] = len(self['pricings']) self['security_contacts_count'] = len(self['security_contacts']) diff --git a/ScoutSuite/providers/azure/resources/securitycenter/security_contacts.py b/ScoutSuite/providers/azure/resources/securitycenter/security_contacts.py index c52b0dfcf..431e502bc 100644 --- a/ScoutSuite/providers/azure/resources/securitycenter/security_contacts.py +++ b/ScoutSuite/providers/azure/resources/securitycenter/security_contacts.py @@ -1,13 +1,14 @@ +from ScoutSuite.providers.azure.facade.facade import AzureFacade from ScoutSuite.providers.base.configs.resources import Resources class SecurityContacts(Resources): - def __init__(self, facade): + def __init__(self, facade: AzureFacade): self.facade = facade async def fetch_all(self): - for raw_contact in await self.facade.get_security_contacts(): + for raw_contact in await self.facade.securitycenter.get_security_contacts(): id, security_contact = self._parse_security_contact(raw_contact) self[id] = security_contact diff --git a/ScoutSuite/providers/azure/resources/sqldatabase/database_blob_auditing_policies.py b/ScoutSuite/providers/azure/resources/sqldatabase/database_blob_auditing_policies.py index d90f75e0f..4fccc4452 100644 --- a/ScoutSuite/providers/azure/resources/sqldatabase/database_blob_auditing_policies.py +++ b/ScoutSuite/providers/azure/resources/sqldatabase/database_blob_auditing_policies.py @@ -1,16 +1,17 @@ +from ScoutSuite.providers.azure.facade.facade import AzureFacade from ScoutSuite.providers.base.configs.resources import Resources class DatabaseBlobAuditingPolicies(Resources): - def __init__(self, resource_group_name, server_name, database_name, facade): + def __init__(self, resource_group_name, server_name, database_name, facade: AzureFacade): self.resource_group_name = resource_group_name self.server_name = server_name self.database_name = database_name self.facade = facade async def fetch_all(self): - policies = await self.facade.get_database_blob_auditing_policies( + policies = await self.facade.sqldatabase.get_database_blob_auditing_policies( self.resource_group_name, self.server_name, self.database_name) self._parse_policies(policies) diff --git a/ScoutSuite/providers/azure/resources/sqldatabase/database_threat_detection_policies.py b/ScoutSuite/providers/azure/resources/sqldatabase/database_threat_detection_policies.py index 9fa2b3225..c22c38bbe 100644 --- a/ScoutSuite/providers/azure/resources/sqldatabase/database_threat_detection_policies.py +++ b/ScoutSuite/providers/azure/resources/sqldatabase/database_threat_detection_policies.py @@ -1,16 +1,17 @@ +from ScoutSuite.providers.azure.facade.facade import AzureFacade from ScoutSuite.providers.base.configs.resources import Resources class DatabaseThreatDetectionPolicies(Resources): - def __init__(self, resource_group_name, server_name, database_name, facade): + def __init__(self, resource_group_name, server_name, database_name, facade: AzureFacade): self.resource_group_name = resource_group_name self.server_name = server_name self.database_name = database_name self.facade = facade async def fetch_all(self): - policies = await self.facade.get_database_threat_detection_policies( + policies = await self.facade.sqldatabase.get_database_threat_detection_policies( self.resource_group_name, self.server_name, self.database_name) self._parse_policies(policies) diff --git a/ScoutSuite/providers/azure/resources/sqldatabase/databases.py b/ScoutSuite/providers/azure/resources/sqldatabase/databases.py index c51de1c92..e2809685b 100644 --- a/ScoutSuite/providers/azure/resources/sqldatabase/databases.py +++ b/ScoutSuite/providers/azure/resources/sqldatabase/databases.py @@ -1,5 +1,6 @@ import asyncio +from ScoutSuite.providers.azure.facade.facade import AzureFacade from ScoutSuite.providers.azure.resources.resources import AzureCompositeResources from .database_blob_auditing_policies import DatabaseBlobAuditingPolicies @@ -16,13 +17,13 @@ class Databases(AzureCompositeResources): (TransparentDataEncryptions, None) ] - def __init__(self, resource_group_name, server_name, facade): + def __init__(self, resource_group_name, server_name, facade: AzureFacade): self.resource_group_name = resource_group_name self.server_name = server_name self.facade = facade async def fetch_all(self): - for db in await self.facade.get_databases(self.resource_group_name, self.server_name): + for db in await self.facade.sqldatabase.get_databases(self.resource_group_name, self.server_name): # We do not want to scan 'master' database which is auto-generated by Azure and read-only: if db.name == 'master': continue diff --git a/ScoutSuite/providers/azure/resources/sqldatabase/replication_links.py b/ScoutSuite/providers/azure/resources/sqldatabase/replication_links.py index bffe27ca8..325c2dd77 100644 --- a/ScoutSuite/providers/azure/resources/sqldatabase/replication_links.py +++ b/ScoutSuite/providers/azure/resources/sqldatabase/replication_links.py @@ -1,16 +1,17 @@ from ScoutSuite.providers.base.configs.resources import Resources +from ScoutSuite.providers.azure.facade.facade import AzureFacade class ReplicationLinks(Resources): - def __init__(self, resource_group_name, server_name, database_name, facade): + def __init__(self, resource_group_name, server_name, database_name, facade: AzureFacade): self.resource_group_name = resource_group_name self.server_name = server_name self.database_name = database_name self.facade = facade async def fetch_all(self): - links = await self.facade.get_database_replication_links( + links = await self.facade.sqldatabase.get_database_replication_links( self.resource_group_name, self.server_name, self.database_name) self._parse_links(links) diff --git a/ScoutSuite/providers/azure/resources/sqldatabase/server_azure_ad_administrators.py b/ScoutSuite/providers/azure/resources/sqldatabase/server_azure_ad_administrators.py index 1552fb98d..a1b6ff45b 100644 --- a/ScoutSuite/providers/azure/resources/sqldatabase/server_azure_ad_administrators.py +++ b/ScoutSuite/providers/azure/resources/sqldatabase/server_azure_ad_administrators.py @@ -1,18 +1,19 @@ from msrestazure.azure_exceptions import CloudError +from ScoutSuite.providers.azure.facade.facade import AzureFacade from ScoutSuite.providers.base.configs.resources import Resources class ServerAzureAdAdministrators(Resources): - def __init__(self, resource_group_name, server_name, facade): + def __init__(self, resource_group_name, server_name, facade: AzureFacade): self.resource_group_name = resource_group_name self.server_name = server_name self.facade = facade async def fetch_all(self): try: - await self.facade.get_server_azure_ad_administrators(self.resource_group_name, self.server_name) + await self.facade.sqldatabase.get_server_azure_ad_administrators(self.resource_group_name, self.server_name) self['ad_admin_configured'] = True except CloudError: # no ad admin configured returns a 404 error self['ad_admin_configured'] = False diff --git a/ScoutSuite/providers/azure/resources/sqldatabase/server_blob_auditing_policies.py b/ScoutSuite/providers/azure/resources/sqldatabase/server_blob_auditing_policies.py index b24e1df90..5983bfd2a 100644 --- a/ScoutSuite/providers/azure/resources/sqldatabase/server_blob_auditing_policies.py +++ b/ScoutSuite/providers/azure/resources/sqldatabase/server_blob_auditing_policies.py @@ -9,7 +9,7 @@ def __init__(self, resource_group_name, server_name, facade): self.facade = facade async def fetch_all(self): - policies = await self.facade.get_server_blob_auditing_policies( + policies = await self.facade.sqldatabase.get_server_blob_auditing_policies( self.resource_group_name, self.server_name) self._parse_policies(policies) diff --git a/ScoutSuite/providers/azure/resources/sqldatabase/server_security_alert_policies.py b/ScoutSuite/providers/azure/resources/sqldatabase/server_security_alert_policies.py index 5d34faed5..5a4ebaae1 100644 --- a/ScoutSuite/providers/azure/resources/sqldatabase/server_security_alert_policies.py +++ b/ScoutSuite/providers/azure/resources/sqldatabase/server_security_alert_policies.py @@ -9,7 +9,7 @@ def __init__(self, resource_group_name, server_name, facade): self.facade = facade async def fetch_all(self): - policies = await self.facade.get_server_security_alert_policies( + policies = await self.facade.sqldatabase.get_server_security_alert_policies( self.resource_group_name, self.server_name) self._parse_policies(policies) diff --git a/ScoutSuite/providers/azure/resources/sqldatabase/servers.py b/ScoutSuite/providers/azure/resources/sqldatabase/servers.py index f2f62e807..c42167810 100644 --- a/ScoutSuite/providers/azure/resources/sqldatabase/servers.py +++ b/ScoutSuite/providers/azure/resources/sqldatabase/servers.py @@ -20,11 +20,8 @@ class Servers(AzureCompositeResources): ] async def fetch_all(self, credentials, **kwargs): - # TODO: build that facade somewhere else: - facade = SQLDatabaseFacade(credentials.credentials, credentials.subscription_id) - self['servers'] = {} - for server in await facade.get_servers(): + for server in await self.facade.sqldatabase.get_servers(): id = get_non_provider_id(server.id) resource_group_name = get_resource_group_name(server.id) @@ -43,7 +40,7 @@ async def fetch_all(self, credentials, **kwargs): parent=server, resource_group_name=server['resource_group_name'], server_name=server['name'], - facade=facade + facade=self.facade ) ) for server in self['servers'].values() } diff --git a/ScoutSuite/providers/azure/resources/sqldatabase/transparent_data_encryptions.py b/ScoutSuite/providers/azure/resources/sqldatabase/transparent_data_encryptions.py index 4fe946a4a..ae743372c 100644 --- a/ScoutSuite/providers/azure/resources/sqldatabase/transparent_data_encryptions.py +++ b/ScoutSuite/providers/azure/resources/sqldatabase/transparent_data_encryptions.py @@ -10,7 +10,7 @@ def __init__(self, resource_group_name, server_name, database_name, facade): self.facade = facade async def fetch_all(self): - encryptions = await self.facade.get_database_transparent_data_encryptions( + encryptions = await self.facade.sqldatabase.get_database_transparent_data_encryptions( self.resource_group_name, self.server_name, self.database_name) self._parse_encryptions(encryptions) diff --git a/ScoutSuite/providers/azure/resources/storageaccounts/blob_containers.py b/ScoutSuite/providers/azure/resources/storageaccounts/blob_containers.py index e8e182dbb..86bcdc3eb 100644 --- a/ScoutSuite/providers/azure/resources/storageaccounts/blob_containers.py +++ b/ScoutSuite/providers/azure/resources/storageaccounts/blob_containers.py @@ -1,14 +1,15 @@ from ScoutSuite.providers.base.configs.resources import Resources +from ScoutSuite.providers.azure.facade.facade import AzureFacade class BlobContainers(Resources): - def __init__(self, resource_group_name, storage_account_name, facade): + def __init__(self, resource_group_name, storage_account_name, facade: AzureFacade): self.resource_group_name = resource_group_name self.storage_account_name = storage_account_name self.facade = facade async def fetch_all(self): - raw_blob_containers = await self.facade.get_blob_containers( + raw_blob_containers = await self.facade.storageaccounts.get_blob_containers( self.resource_group_name, self.storage_account_name ) for raw_blob_container in raw_blob_containers: diff --git a/ScoutSuite/providers/azure/resources/storageaccounts/storageaccounts.py b/ScoutSuite/providers/azure/resources/storageaccounts/storageaccounts.py index 7dfe72ad9..c361aa3bf 100644 --- a/ScoutSuite/providers/azure/resources/storageaccounts/storageaccounts.py +++ b/ScoutSuite/providers/azure/resources/storageaccounts/storageaccounts.py @@ -3,7 +3,6 @@ from ScoutSuite.providers.azure.resources.resources import AzureCompositeResources from ScoutSuite.providers.azure.utils import get_resource_group_name from ScoutSuite.providers.utils import get_non_provider_id -from ScoutSuite.providers.azure.facade.storageaccounts import StorageAccountsFacade from .blob_containers import BlobContainers @@ -14,11 +13,8 @@ class StorageAccounts(AzureCompositeResources): ] async def fetch_all(self, credentials, **kwargs): - # TODO: build that facade somewhere else: - facade = StorageAccountsFacade(credentials.credentials, credentials.subscription_id) - self['storage_accounts'] = {} - for raw_storage_account in await facade.get_storage_accounts(): + for raw_storage_account in await self.facade.storageaccounts.get_storage_accounts(): id, storage_account = self._parse_storage_account(raw_storage_account) self['storage_accounts'][id] = storage_account @@ -31,7 +27,7 @@ async def fetch_all(self, credentials, **kwargs): parent=storage_account, resource_group_name=storage_account['resource_group_name'], storage_account_name=storage_account['name'], - facade=facade + facade=self.facade ) ) for storage_account in self['storage_accounts'].values() } diff --git a/ScoutSuite/providers/gcp/configs/services.py b/ScoutSuite/providers/gcp/configs/services.py index 0b99426b7..79311ea6b 100644 --- a/ScoutSuite/providers/gcp/configs/services.py +++ b/ScoutSuite/providers/gcp/configs/services.py @@ -28,7 +28,7 @@ def __init__(self, credentials=None, thread_config=4, projects=None, **kwargs): try: self.kubernetesengine = KubernetesEngineConfig(thread_config=thread_config) - except NameError as e: + except NameError as _: pass self.stackdriverlogging = StackdriverLoggingConfig(thread_config=thread_config) From c1c9906a1c6ce196aef2057d9c9858377936ebab Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 27 Mar 2019 20:15:50 -0400 Subject: [PATCH 627/667] Removed authenticate method --- ScoutSuite/providers/aws/provider.py | 8 --- ScoutSuite/providers/azure/provider.py | 91 -------------------------- ScoutSuite/providers/base/provider.py | 7 -- ScoutSuite/providers/gcp/provider.py | 3 - 4 files changed, 109 deletions(-) diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index a76913be3..0cfa4b5ee 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -41,14 +41,6 @@ def __init__(self, profile='default', report_dir=None, timestamp=None, services= super(AWSProvider, self).__init__(report_dir, timestamp, services, skipped_services, thread_config) - - def authenticate(self, profile=None, **kwargs): - """ - Implement authentication for the AWS provider - :return: - """ - return True - def preprocessing(self, ip_ranges=None, ip_ranges_name_key=None): """ Tweak the AWS config to match cross-service resources and clean any fetching artifacts diff --git a/ScoutSuite/providers/azure/provider.py b/ScoutSuite/providers/azure/provider.py index 370795eea..da22e6099 100644 --- a/ScoutSuite/providers/azure/provider.py +++ b/ScoutSuite/providers/azure/provider.py @@ -33,97 +33,6 @@ def __init__(self, project_id=None, organization_id=None, super(AzureProvider, self).__init__(report_dir, timestamp, services, skipped_services, thread_config) - def authenticate(self, cli=None, msi=None, service_principal=None, file_auth=None, user_account=None, - tenant_id=None, subscription_id=None, client_id=None, client_secret=None, username=None, - password=None, **kargs): - """ - Implements authentication for the Azure provider using azure-cli. - Refer to https://docs.microsoft.com/en-us/python/azure/python-sdk-azure-authenticate?view=azure-python. - - :return: - """ - - try: - if cli: - cli_credentials, self.aws_account_id = get_azure_cli_credentials() # TODO: Remove aws_account_id - self.credentials = AzureCredentials(cli_credentials, self.aws_account_id) - return True - elif msi: - credentials = MSIAuthentication() - - # Get the subscription ID - subscription_client = SubscriptionClient(credentials) - try: - # Tries to read the subscription list - subscription = next(subscription_client.subscriptions.list()) - self.aws_account_id = subscription.subscription_id - except StopIteration: - # If the VM cannot read subscription list, ask Subscription ID: - self.aws_account_id = input('Subscription ID: ') - - self.credentials = AzureCredentials(credentials, self.aws_account_id) - return True - elif file_auth: - data = json.loads(file_auth.read()) - subscription_id = data.get('subscriptionId') - tenant_id = data.get('tenantId') - client_id = data.get('clientId') - client_secret = data.get('clientSecret') - - self.aws_account_id = tenant_id # TODO this is for AWS - - credentials = ServicePrincipalCredentials( - client_id=client_id, - secret=client_secret, - tenant=tenant_id - ) - - self.credentials = AzureCredentials(credentials, subscription_id) - - return True - elif service_principal: - subscription_id = subscription_id if subscription_id else input("Subscription ID: ") - tenant_id = tenant_id if tenant_id else input("Tenant ID: ") - client_id = client_id if client_id else input("Client ID: ") - client_secret = client_secret if client_secret else getpass("Client secret: ") - - self.aws_account_id = tenant_id # TODO this is for AWS - - credentials = ServicePrincipalCredentials( - client_id=client_id, - secret=client_secret, - tenant=tenant_id - ) - - self.credentials = AzureCredentials(credentials, subscription_id) - - return True - elif user_account: - username = username if username else input("Username: ") - password = password if password else getpass("Password: ") - - credentials = UserPassCredentials(username, password) - - if subscription_id: - self.aws_account_id = subscription_id - else: - # Get the subscription ID - subscription_client = SubscriptionClient(credentials) - try: - # Tries to read the subscription list - subscription = next(subscription_client.subscriptions.list()) - self.aws_account_id = subscription.subscription_id - except StopIteration: - # If the user cannot read subscription list, ask Subscription ID: - self.aws_account_id = input('Subscription ID: ') - - self.credentials = AzureCredentials(credentials, self.aws_account_id) - return True - except Exception as e: - print_error('Failed to authenticate to Azure') - print_exception(e) - return False - def preprocessing(self, ip_ranges=None, ip_ranges_name_key=None): """ TODO description diff --git a/ScoutSuite/providers/base/provider.py b/ScoutSuite/providers/base/provider.py index b081be713..1181ebd0c 100644 --- a/ScoutSuite/providers/base/provider.py +++ b/ScoutSuite/providers/base/provider.py @@ -46,13 +46,6 @@ def __init__(self, report_dir=None, timestamp=None, services=None, skipped_servi supported_services = vars(self.services).keys() self.service_list = self._build_services_list(supported_services, services, skipped_services) - def authenticate(self): - """ - Authenticate to the provider using provided credentials - :return: - """ - pass - def preprocessing(self, ip_ranges=None, ip_ranges_name_key=None): """ TODO diff --git a/ScoutSuite/providers/gcp/provider.py b/ScoutSuite/providers/gcp/provider.py index 92f590b1d..75980bbd3 100644 --- a/ScoutSuite/providers/gcp/provider.py +++ b/ScoutSuite/providers/gcp/provider.py @@ -43,9 +43,6 @@ def __init__(self, project_id=None, folder_id=None, organization_id=None, all_pr super(GCPProvider, self).__init__(report_dir, timestamp, services, skipped_services, thread_config) - def authenticate(self, user_account=None, service_account=None, **kargs): - pass - def _set_aws_account_id(self): if self.all_projects: if self.credentials.is_service_account and hasattr(self.credentials, 'service_account_email'): From e7ca4889260c6edd4b551ff0455d36a916e7702d Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 27 Mar 2019 20:28:04 -0400 Subject: [PATCH 628/667] Lint --- ScoutSuite/providers/aws/configs/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index 9a7145230..b80e1aa8d 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -47,7 +47,7 @@ class AWSServicesConfig(BaseServicesConfig): :ivar rds: RDS configuration :ivar redshift: Redshift configuration :ivar s3: S3 configuration - :ivar ses: SES configuration: TODO + :ivar ses: SES configuration: "ivar sns: SNS configuration :ivar sqs: SQS configuration """ From 54efb8582733cded339ab38bcb9c9634b0383ff2 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 27 Mar 2019 20:28:15 -0400 Subject: [PATCH 629/667] Removed unused file --- ScoutSuite/providers/azure/configs/base.py | 45 ---------------------- 1 file changed, 45 deletions(-) delete mode 100644 ScoutSuite/providers/azure/configs/base.py diff --git a/ScoutSuite/providers/azure/configs/base.py b/ScoutSuite/providers/azure/configs/base.py deleted file mode 100644 index a1a2683aa..000000000 --- a/ScoutSuite/providers/azure/configs/base.py +++ /dev/null @@ -1,45 +0,0 @@ -# -*- coding: utf-8 -*- - -# Python2 vs Python3 -try: - from Queue import Queue -except ImportError: - from queue import Queue - -from ScoutSuite.providers.base.configs.base import BaseConfig - - -class AzureBaseConfig(BaseConfig): - - def __init__(self, thread_config=4, **kwargs): - - super(AzureBaseConfig, self).__init__(thread_config) - - def _is_provider(self, provider_name): - return provider_name == 'azure' - - def _get_method(self, api_client, target_type, list_method_name): - """ - Gets the appropriate method, required as each provider may have particularities - - :return: - """ - - target = getattr(api_client, target_type) - method = getattr(target, list_method_name) - - return method - - def _get_targets(self, response_attribute, api_client, method, list_params, ignore_list_error): - """ - Fetch the targets, required as each provider may have particularities - - :return: - """ - - targets = [] - response = method(**list_params) - for i in response: - targets.append(i) - - return targets From 39bd60ba9f8e2e7d91cd53220db9fc9dc2e2116e Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 27 Mar 2019 20:39:42 -0400 Subject: [PATCH 630/667] Saved credentials in base services config --- ScoutSuite/providers/aws/configs/services.py | 4 ++-- ScoutSuite/providers/azure/configs/services.py | 3 ++- ScoutSuite/providers/base/configs/services.py | 10 +++++----- ScoutSuite/providers/base/provider.py | 2 +- ScoutSuite/providers/gcp/configs/services.py | 3 ++- 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index b80e1aa8d..5d0bcd0bb 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -52,8 +52,8 @@ class AWSServicesConfig(BaseServicesConfig): :ivar sqs: SQS configuration """ - def __init__(self, credentials=None, thread_config=4, **kwargs): - super(AWSServicesConfig, self).__init__(credentials, thread_config) + def __init__(self, credentials=None, **kwargs): + super(AWSServicesConfig, self).__init__(credentials) facade = AWSFacade(credentials) diff --git a/ScoutSuite/providers/azure/configs/services.py b/ScoutSuite/providers/azure/configs/services.py index 02fccae6b..60cecdf93 100644 --- a/ScoutSuite/providers/azure/configs/services.py +++ b/ScoutSuite/providers/azure/configs/services.py @@ -29,7 +29,8 @@ class AzureServicesConfig(BaseServicesConfig): - def __init__(self, credentials: AzureCredentials = None, thread_config=4, **kwargs): + def __init__(self, credentials: AzureCredentials = None, **kwargs): + super(AzureServicesConfig, self).__init__(credentials) facade = AzureFacade(credentials) self.storageaccounts = StorageAccounts(facade) diff --git a/ScoutSuite/providers/base/configs/services.py b/ScoutSuite/providers/base/configs/services.py index 4ee4dca39..4452a2efa 100644 --- a/ScoutSuite/providers/base/configs/services.py +++ b/ScoutSuite/providers/base/configs/services.py @@ -7,13 +7,13 @@ class BaseServicesConfig(object): - def __init__(self, credentials=None, thread_config=4): - pass + def __init__(self, credentials): + self.credentials = credentials def _is_provider(self, provider_name): return False - async def fetch(self, credentials, services=None, regions=None): + async def fetch(self, services=None, regions=None): services = [] if services is None else services regions = [] if regions is None else regions for service in vars(self): @@ -26,11 +26,11 @@ async def fetch(self, credentials, services=None, regions=None): service_config = getattr(self, service) # call fetch method for the service if 'fetch_all' in dir(service_config): - method_args = {'credentials': credentials, 'regions': regions} + method_args = {'credentials': self.credentials, 'regions': regions} if self._is_provider('aws'): if service != 'iam': - method_args['partition_name'] = get_partition_name(credentials) + method_args['partition_name'] = get_partition_name(self.credentials) await service_config.fetch_all(**method_args) if hasattr(service_config, 'finalize'): diff --git a/ScoutSuite/providers/base/provider.py b/ScoutSuite/providers/base/provider.py index 1181ebd0c..29dcfcac1 100644 --- a/ScoutSuite/providers/base/provider.py +++ b/ScoutSuite/providers/base/provider.py @@ -80,7 +80,7 @@ async def fetch(self, regions=None, skipped_regions=None, partition_name=None): regions = [] if regions is None else regions skipped_regions = [] if skipped_regions is None else skipped_regions # TODO: determine partition name based on regions and warn if multiple partitions... - await self.services.fetch(self.credentials, self.service_list, regions) + await self.services.fetch(self.service_list, regions) # TODO implement this properly """ diff --git a/ScoutSuite/providers/gcp/configs/services.py b/ScoutSuite/providers/gcp/configs/services.py index 79311ea6b..58b2f4f07 100644 --- a/ScoutSuite/providers/gcp/configs/services.py +++ b/ScoutSuite/providers/gcp/configs/services.py @@ -17,7 +17,8 @@ class GCPServicesConfig(BaseServicesConfig): def __init__(self, credentials=None, thread_config=4, projects=None, **kwargs): - + super(GCPServicesConfig, self).__init__(credentials) + projects = [] if projects is None else projects self.cloudresourcemanager = CloudResourceManager(thread_config=thread_config) From 13d7aecbcd2181263e3495ef3fb5612453df41d4 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 27 Mar 2019 20:39:49 -0400 Subject: [PATCH 631/667] Added newline --- ScoutSuite/providers/base/configs/services.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ScoutSuite/providers/base/configs/services.py b/ScoutSuite/providers/base/configs/services.py index 4452a2efa..3ab416935 100644 --- a/ScoutSuite/providers/base/configs/services.py +++ b/ScoutSuite/providers/base/configs/services.py @@ -22,6 +22,7 @@ async def fetch(self, services=None, regions=None): if services != [] and service not in services: print_debug('Skipping the {} service'.format(format_service_name(service))) continue + print_info('Fetching resources for the {} service'.format(format_service_name(service))) service_config = getattr(self, service) # call fetch method for the service From c7aed145ceef8a79e4c520435783a077ac4d194f Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 27 Mar 2019 20:43:31 -0400 Subject: [PATCH 632/667] Added try around private imports --- ScoutSuite/providers/azure/facade/facade.py | 22 ++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/ScoutSuite/providers/azure/facade/facade.py b/ScoutSuite/providers/azure/facade/facade.py index 2ca639e24..3b1f78c00 100644 --- a/ScoutSuite/providers/azure/facade/facade.py +++ b/ScoutSuite/providers/azure/facade/facade.py @@ -1,7 +1,4 @@ -from ScoutSuite.providers.azure.facade.appgateway_private import AppGatewayFacade -from ScoutSuite.providers.azure.facade.appservice_private import AppServiceFacade from ScoutSuite.providers.azure.facade.keyvault import KeyVaultFacade -from ScoutSuite.providers.azure.facade.loadbalancer_private import LoadBalancerFacade from ScoutSuite.providers.azure.facade.network import NetworkFacade from ScoutSuite.providers.azure.facade.rediscache_private import RedisCacheFacade from ScoutSuite.providers.azure.facade.securitycenter import SecurityCenterFacade @@ -9,15 +6,26 @@ from ScoutSuite.providers.azure.facade.storageaccounts import StorageAccountsFacade from ScoutSuite.providers.azure.authentication_strategy import AzureCredentials +try: + from ScoutSuite.providers.azure.facade.appgateway_private import AppGatewayFacade + from ScoutSuite.providers.azure.facade.appservice_private import AppServiceFacade + from ScoutSuite.providers.azure.facade.loadbalancer_private import LoadBalancerFacade +except ImportError: + pass + class AzureFacade(): def __init__(self, credentials: AzureCredentials): - self.appgateway = AppGatewayFacade(credentials.credentials, credentials.subscription_id) - self.appservice = AppServiceFacade(credentials.credentials, credentials.subscription_id) self.keyvault = KeyVaultFacade(credentials.credentials, credentials.subscription_id) - self.loadbalancer = LoadBalancerFacade(credentials.credentials, credentials.subscription_id) self.network = NetworkFacade(credentials.credentials, credentials.subscription_id) self.rediscache = RedisCacheFacade(credentials.credentials, credentials.subscription_id) self.securitycenter = SecurityCenterFacade(credentials.credentials, credentials.subscription_id) self.sqldatabase = SQLDatabaseFacade(credentials.credentials, credentials.subscription_id) - self.storageaccounts = StorageAccountsFacade(credentials.credentials, credentials.subscription_id) \ No newline at end of file + self.storageaccounts = StorageAccountsFacade(credentials.credentials, credentials.subscription_id) + + try: + self.appgateway = AppGatewayFacade(credentials.credentials, credentials.subscription_id) + self.appservice = AppServiceFacade(credentials.credentials, credentials.subscription_id) + self.loadbalancer = LoadBalancerFacade(credentials.credentials, credentials.subscription_id) + except NameError: + pass \ No newline at end of file From 686c065bfabf8ec3ae104bf1d697099836b02b18 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 27 Mar 2019 20:44:40 -0400 Subject: [PATCH 633/667] Removed thread configs argument --- ScoutSuite/providers/base/provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/base/provider.py b/ScoutSuite/providers/base/provider.py index 29dcfcac1..978a91bd3 100644 --- a/ScoutSuite/providers/base/provider.py +++ b/ScoutSuite/providers/base/provider.py @@ -42,7 +42,7 @@ def __init__(self, report_dir=None, timestamp=None, services=None, skipped_servi self._load_metadata() - self.services = self.services_config(self.credentials, thread_config) + self.services = self.services_config(self.credentials) supported_services = vars(self.services).keys() self.service_list = self._build_services_list(supported_services, services, skipped_services) From 0ca94658016c0db781c8091a59aac9d298f22b36 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Wed, 27 Mar 2019 20:52:25 -0400 Subject: [PATCH 634/667] Moved redis cache facade in try --- ScoutSuite/providers/aws/resources/elasticache/service.py | 6 ++---- ScoutSuite/providers/azure/facade/facade.py | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/elasticache/service.py b/ScoutSuite/providers/aws/resources/elasticache/service.py index 36f52a04c..ff9e8e283 100644 --- a/ScoutSuite/providers/aws/resources/elasticache/service.py +++ b/ScoutSuite/providers/aws/resources/elasticache/service.py @@ -1,8 +1,6 @@ from ScoutSuite.providers.aws.facade.facade import AWSFacade -from ScoutSuite.providers.aws.resources.elasticache.parametergroups import \ - ParameterGroups -from ScoutSuite.providers.aws.resources.elasticache.securitygroups import \ - SecurityGroups +from ScoutSuite.providers.aws.resources.elasticache.parametergroups import ParameterGroups +from ScoutSuite.providers.aws.resources.elasticache.securitygroups import SecurityGroups from ScoutSuite.providers.aws.resources.elasticache.vpcs import ElastiCacheVpcs from ScoutSuite.providers.aws.resources.regions import Regions diff --git a/ScoutSuite/providers/azure/facade/facade.py b/ScoutSuite/providers/azure/facade/facade.py index 3b1f78c00..9344db261 100644 --- a/ScoutSuite/providers/azure/facade/facade.py +++ b/ScoutSuite/providers/azure/facade/facade.py @@ -1,6 +1,5 @@ from ScoutSuite.providers.azure.facade.keyvault import KeyVaultFacade from ScoutSuite.providers.azure.facade.network import NetworkFacade -from ScoutSuite.providers.azure.facade.rediscache_private import RedisCacheFacade from ScoutSuite.providers.azure.facade.securitycenter import SecurityCenterFacade from ScoutSuite.providers.azure.facade.sqldatabase import SQLDatabaseFacade from ScoutSuite.providers.azure.facade.storageaccounts import StorageAccountsFacade @@ -10,6 +9,7 @@ from ScoutSuite.providers.azure.facade.appgateway_private import AppGatewayFacade from ScoutSuite.providers.azure.facade.appservice_private import AppServiceFacade from ScoutSuite.providers.azure.facade.loadbalancer_private import LoadBalancerFacade + from ScoutSuite.providers.azure.facade.rediscache_private import RedisCacheFacade except ImportError: pass @@ -18,7 +18,6 @@ class AzureFacade(): def __init__(self, credentials: AzureCredentials): self.keyvault = KeyVaultFacade(credentials.credentials, credentials.subscription_id) self.network = NetworkFacade(credentials.credentials, credentials.subscription_id) - self.rediscache = RedisCacheFacade(credentials.credentials, credentials.subscription_id) self.securitycenter = SecurityCenterFacade(credentials.credentials, credentials.subscription_id) self.sqldatabase = SQLDatabaseFacade(credentials.credentials, credentials.subscription_id) self.storageaccounts = StorageAccountsFacade(credentials.credentials, credentials.subscription_id) @@ -27,5 +26,6 @@ def __init__(self, credentials: AzureCredentials): self.appgateway = AppGatewayFacade(credentials.credentials, credentials.subscription_id) self.appservice = AppServiceFacade(credentials.credentials, credentials.subscription_id) self.loadbalancer = LoadBalancerFacade(credentials.credentials, credentials.subscription_id) + self.rediscache = RedisCacheFacade(credentials.credentials, credentials.subscription_id) except NameError: pass \ No newline at end of file From 74204e17e1be2bfe6f9e9c2b9101a456b1d16e9f Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Thu, 28 Mar 2019 08:48:10 -0400 Subject: [PATCH 635/667] Removed commented line --- ScoutSuite/providers/aws/resources/s3/service.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/s3/service.py b/ScoutSuite/providers/aws/resources/s3/service.py index a77dafe31..bcacd7064 100644 --- a/ScoutSuite/providers/aws/resources/s3/service.py +++ b/ScoutSuite/providers/aws/resources/s3/service.py @@ -27,11 +27,9 @@ def _parse_bucket(self, raw_bucket): :return: """ raw_bucket['name'] = raw_bucket.pop('Name') - # api_client = params['api_clients'][get_s3_list_region(list(params['api_clients'].keys())[0])] - raw_bucket['CreationDate'] = str(raw_bucket['CreationDate']) - # # If requested, get key properties + # If requested, get key properties raw_bucket['id'] = get_non_provider_id(raw_bucket['name']) return raw_bucket['id'], raw_bucket From a95dc2d9b5f93f98b845e9abfdf5bf5c44c20a60 Mon Sep 17 00:00:00 2001 From: misg Date: Thu, 28 Mar 2019 09:53:18 -0400 Subject: [PATCH 636/667] Fix test. --- tests/test-aws_resources.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test-aws_resources.py b/tests/test-aws_resources.py index ccb99852f..32b80e402 100644 --- a/tests/test-aws_resources.py +++ b/tests/test-aws_resources.py @@ -4,11 +4,13 @@ import json import os + class DummyResources(AWSResources): async def fetch_all(self): self['resource_a'] = { 'some_id': 1, 'from_scope': self.scope } self['resource_b'] = { 'some_id': 2, 'from_scope': self.scope } + class DummyComposite(AWSCompositeResources): _children = [ (DummyResources, 'some_dummy_resources'), @@ -25,7 +27,7 @@ async def fetch_all(self, **kwargs): 'some_inner_scope': key } - await self._fetch_children(parent=self[key], scope=scope) + await self._fetch_children(resource_parent=self[key], scope=scope) class TestAWSResources(TestCase): @@ -43,4 +45,3 @@ def test_aws_composite_resource(self): actual_json = json.dumps(composite) assert (expected_json == actual_json) - From 5aadef624133231a1dd65021f89dc158934cba33 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Thu, 28 Mar 2019 10:15:55 -0400 Subject: [PATCH 637/667] Made number of threads configurable --- Scout.py | 10 +--------- ScoutSuite/__main__.py | 27 +++++++++++++++++++-------- ScoutSuite/core/cli_parser.py | 5 +++++ 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/Scout.py b/Scout.py index 1f1800adf..22cebc171 100755 --- a/Scout.py +++ b/Scout.py @@ -2,16 +2,8 @@ # -*- coding: utf-8 -*- import sys -import asyncio - -from concurrent.futures import ThreadPoolExecutor - from ScoutSuite.__main__ import main if __name__ == "__main__": - loop = asyncio.get_event_loop() - # TODO: make max_workers parameterizable (through the thread_config cli argument) - loop.set_default_executor(ThreadPoolExecutor(max_workers=10)) - loop.run_until_complete(main()) - loop.close() + main() sys.exit() diff --git a/ScoutSuite/__main__.py b/ScoutSuite/__main__.py index e3b0afd79..ff75846f1 100644 --- a/ScoutSuite/__main__.py +++ b/ScoutSuite/__main__.py @@ -1,7 +1,9 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +import asyncio import copy +from concurrent.futures import ThreadPoolExecutor import os import webbrowser @@ -16,8 +18,7 @@ from ScoutSuite.providers.base.authentication_strategy_factory import get_authentication_strategy -# noinspection PyBroadException -async def main(args=None): +def main(args=None): """ Main method that runs a scan """ @@ -29,6 +30,13 @@ async def main(args=None): # Get the dictionnary to get None instead of a crash args = args.__dict__ + loop = asyncio.get_event_loop() + loop.set_default_executor(ThreadPoolExecutor(max_workers=args.get('max_workers'))) + loop.run_until_complete(run_scan(args)) + loop.close() + +# noinspection PyBroadException +async def run_scan(args): # Configure the debug level set_config_debug_level(args.get('debug')) @@ -54,7 +62,6 @@ async def main(args=None): if not credentials: return 401 - # Create a cloud provider object cloud_provider = get_provider(provider=args.get('provider'), profile=args.get('profile'), @@ -65,7 +72,8 @@ async def main(args=None): report_dir=args.get('report_dir'), timestamp=args.get('timestamp'), services=args.get('services'), - skipped_services=args.get('skipped_services'), + skipped_services=args.get( + 'skipped_services'), thread_config=args.get('thread_config'), credentials=credentials) @@ -74,7 +82,8 @@ async def main(args=None): # TODO: move this to after authentication, so that the report can be more specific to what's being scanned. # For example if scanning with a GCP service account, the SA email can only be known after authenticating... # Create a new report - report = Scout2Report(args.get('provider'), report_file_name, args.get('report_dir'), args.get('timestamp')) + report = Scout2Report(args.get('provider'), report_file_name, args.get( + 'report_dir'), args.get('timestamp')) # Complete run, including pulling data from provider if not args.get('fetch_local'): @@ -129,14 +138,16 @@ async def main(args=None): if args.get('exceptions')[0]: print_info('Applying exceptions') try: - exceptions = RuleExceptions(args.get('profile'), args.get('exceptions')[0]) + exceptions = RuleExceptions( + args.get('profile'), args.get('exceptions')[0]) exceptions.process(cloud_provider) exceptions = exceptions.exceptions except Exception as e: - print_debug('Failed to load exceptions. The file may not exist or may have an invalid format.') + print_debug( + 'Failed to load exceptions. The file may not exist or may have an invalid format.') exceptions = {} else: - exceptions = {} + exceptions = {} # Handle exceptions try: exceptions = RuleExceptions( diff --git a/ScoutSuite/core/cli_parser.py b/ScoutSuite/core/cli_parser.py index 0ec89f78e..ef54265ed 100644 --- a/ScoutSuite/core/cli_parser.py +++ b/ScoutSuite/core/cli_parser.py @@ -203,11 +203,16 @@ def _init_common_args_parser(self): default=False, action='store_true', help='Do not automatically open the report in the browser.') + # TODO: This should be removed once the GCP refactoring is done parser.add_argument('--thread-config', dest='thread_config', type=int, default=4, help='Level of multi-threading wanted [1-5]; defaults to 4.') + parser.add_argument('--max-workers', + dest='max_workers', + type=int, + help='Maximum number of threads used by Scout Suite') parser.add_argument('--report-dir', dest='report_dir', default=DEFAULT_REPORT_DIR, From 0ad45f2feb61c1d1796b9a11dff799add35819d4 Mon Sep 17 00:00:00 2001 From: misg Date: Thu, 28 Mar 2019 11:44:46 -0400 Subject: [PATCH 638/667] Revert "Parallelized loadbalancers fetching" This reverts commit c60a67d3 --- .../aws/resources/elbv2/load_balancers.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/ScoutSuite/providers/aws/resources/elbv2/load_balancers.py b/ScoutSuite/providers/aws/resources/elbv2/load_balancers.py index 41ba05676..e67ca7f5a 100644 --- a/ScoutSuite/providers/aws/resources/elbv2/load_balancers.py +++ b/ScoutSuite/providers/aws/resources/elbv2/load_balancers.py @@ -1,7 +1,6 @@ -import asyncio - from ScoutSuite.providers.aws.resources.resources import AWSCompositeResources from ScoutSuite.providers.utils import get_non_provider_id + from .listeners import Listeners @@ -12,13 +11,12 @@ class LoadBalancers(AWSCompositeResources): async def fetch_all(self, **kwargs): raw_loads_balancers = await self.facade.elbv2.get_load_balancers(self.scope['region'], self.scope['vpc']) - await asyncio.wait({asyncio.ensure_future(self._fetch_one(raw_load_balancer)) for raw_load_balancer in raw_loads_balancers}) - - async def _fetch_one(self, raw_load_balancer): - id, load_balancer = await self._parse_load_balancer(raw_load_balancer) - self[id] = load_balancer - await self._fetch_children(parent=load_balancer, scope={'region': self.scope['region'], 'load_balancer_arn': load_balancer['arn']}) - + # TODO: parallelize the following loop which is async: + for raw_load_balancer in raw_loads_balancers: + id, load_balancer = await self._parse_load_balancer(raw_load_balancer) + self[id] = load_balancer + await self._fetch_children( + parent=load_balancer, scope={'region': self.scope['region'], 'load_balancer_arn': load_balancer['arn']}) async def _parse_load_balancer(self, load_balancer): load_balancer['arn'] = load_balancer.pop('LoadBalancerArn') From d24107ec1a52f7e27b6b2a15f5bb8e65d59cb3f4 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Fri, 29 Mar 2019 09:05:52 -0400 Subject: [PATCH 639/667] Update cli_parser.py --- ScoutSuite/core/cli_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/core/cli_parser.py b/ScoutSuite/core/cli_parser.py index ef54265ed..42bcf2a73 100644 --- a/ScoutSuite/core/cli_parser.py +++ b/ScoutSuite/core/cli_parser.py @@ -212,7 +212,7 @@ def _init_common_args_parser(self): parser.add_argument('--max-workers', dest='max_workers', type=int, - help='Maximum number of threads used by Scout Suite') + help='Maximum number of threads (workers) used by Scout Suite') parser.add_argument('--report-dir', dest='report_dir', default=DEFAULT_REPORT_DIR, From d3886ec00ada8c57d1722f6d152f2b578d346ddf Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Fri, 29 Mar 2019 09:07:20 -0400 Subject: [PATCH 640/667] Set default number of threads to 10 --- ScoutSuite/core/cli_parser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ScoutSuite/core/cli_parser.py b/ScoutSuite/core/cli_parser.py index 42bcf2a73..e901bbb00 100644 --- a/ScoutSuite/core/cli_parser.py +++ b/ScoutSuite/core/cli_parser.py @@ -212,6 +212,7 @@ def _init_common_args_parser(self): parser.add_argument('--max-workers', dest='max_workers', type=int, + default=10, help='Maximum number of threads (workers) used by Scout Suite') parser.add_argument('--report-dir', dest='report_dir', From bf98debab5a2943062eb950cf6301ab4be9fbc97 Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 30 Mar 2019 15:13:00 -0400 Subject: [PATCH 641/667] Define get_and_set_concurrently function and fix PEP8 issues. --- ScoutSuite/providers/aws/facade/utils.py | 42 +++++++++++++++++++++--- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/utils.py b/ScoutSuite/providers/aws/facade/utils.py index 4649d66b3..86193adb8 100644 --- a/ScoutSuite/providers/aws/facade/utils.py +++ b/ScoutSuite/providers/aws/facade/utils.py @@ -1,4 +1,5 @@ import boto3 +import asyncio from ScoutSuite.providers.utils import run_concurrently @@ -7,7 +8,8 @@ class AWSFacadeUtils: _clients = {} @staticmethod - async def get_all_pages(service: str, region: str, session: boto3.session.Session, paginator_name: str, entity: str, **paginator_args): + async def get_all_pages(service: str, region: str, session: boto3.session.Session, paginator_name: str, + entity: str, **paginator_args): """ Gets all the entities from a paginator given an entity key @@ -21,11 +23,14 @@ async def get_all_pages(service: str, region: str, session: boto3.session.Sessio :return: A list of the fetched entities. """ - results = await AWSFacadeUtils.get_multiple_entities_from_all_pages(service, region, session, paginator_name, [entity], **paginator_args) + results = await AWSFacadeUtils.get_multiple_entities_from_all_pages( + service, region, session, paginator_name,[entity], **paginator_args) + return results[entity] @staticmethod - async def get_multiple_entities_from_all_pages(service: str, region: str, session: boto3.session.Session, paginator_name: str, entities: list, **paginator_args): + async def get_multiple_entities_from_all_pages(service: str, region: str, session: boto3.session.Session, + paginator_name: str, entities: list, **paginator_args): """ Gets all the entities from a paginator given multiple entitiy keys :param service:str: Name of the AWS service (ec2, iam, etc.) @@ -69,5 +74,32 @@ def get_client(service: str, session: boto3.session.Session, region: str = None) :return: """ - return AWSFacadeUtils._clients.setdefault((service, region), - session.client(service, region_name=region) if region else session.client(service)) + + return AWSFacadeUtils._clients.setdefault( + (service, region), + session.client(service, region_name=region) if region else session.client(service)) + + @staticmethod + async def get_and_set_concurrently(get_and_set_funcs: [], entities: [], **kwargs): + """ + Given a list of get_and_set_* functions (ex: get_and_set_description, get_and_set_attributes, + get_and_set_policy, etc.) and a list of entities (ex: stacks, keys, load balancers, vpcs, etc.), + get_and_set_concurrently will call each of these functions concurrently on each entity. + + :param get_and_set_funcs: list of functions that takes a region and an entity (they must have the following + signature: region: str, entity: {}) and then fetch and set some kind of attributes to this entity. + :param entities: list of a same kind of entities + :param region: a region + + :return: + """ + + if len(entities) == 0: + return + + tasks = { + asyncio.ensure_future( + get_and_set_func(entity, **kwargs) + ) for entity in entities for get_and_set_func in get_and_set_funcs + } + await asyncio.wait(tasks) From 60195310226c5328b12864c61e3de6ea013267c8 Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 30 Mar 2019 15:13:52 -0400 Subject: [PATCH 642/667] Define map_concurrently function. --- ScoutSuite/providers/utils.py | 42 ++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/ScoutSuite/providers/utils.py b/ScoutSuite/providers/utils.py index 14f09db60..bcedcc3d7 100644 --- a/ScoutSuite/providers/utils.py +++ b/ScoutSuite/providers/utils.py @@ -1,6 +1,7 @@ -from hashlib import sha1 import asyncio +from hashlib import sha1 + def get_non_provider_id(name): """ @@ -16,17 +17,36 @@ def get_non_provider_id(name): def run_concurrently(func): - return asyncio.get_event_loop().run_in_executor(executor=None, func=func) + """ + Schedules the execution of function `func` in the default thread pool (referred as 'executor') that has been + associated with the global event loop. -async def run_tasks_concurrently(tasks): - entities = [] + :param func: function to be executed concurrently, in a dedicated thread. + :return: an asyncio.Future to be awaited. + """ + + return asyncio.get_event_loop().run_in_executor(executor=None, func=func) - parallelized_tasks = { - asyncio.ensure_future(task) for task in tasks - } - for task in asyncio.as_completed(parallelized_tasks): - entity = await task - entities.append(entity) +async def map_concurrently(coro, entities, **kwargs): + """ + Given a list of entities, executes coroutine `coro` concurrently on each entity and returns a list of the obtained + results ([await coro(entity_x), await coro(entity_a), ..., await coro(entity_z)]). + + :param coro: coroutine to be executed concurrently. Takes an entity as parameter and returns a new entity. + :param entities: a list of the same type of entity (ex: cluster ids) + + :return: a list of new entities (ex: clusters) + """ + results = [] + + tasks = { + asyncio.ensure_future( + coro(entity, **kwargs) + ) for entity in entities + } + for task in asyncio.as_completed(tasks): + result = await task + results.append(result) - return entities + return results From 0537bb203163c2f4921e967a12e9fd0d411d8170 Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 30 Mar 2019 15:14:39 -0400 Subject: [PATCH 643/667] Improve concurrent support for the CloudFormation facade. --- .../providers/aws/facade/cloudformation.py | 43 +++++++++++-------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/cloudformation.py b/ScoutSuite/providers/aws/facade/cloudformation.py index db7e04f64..be2f8980c 100644 --- a/ScoutSuite/providers/aws/facade/cloudformation.py +++ b/ScoutSuite/providers/aws/facade/cloudformation.py @@ -6,28 +6,35 @@ class CloudFormation(AWSBaseFacade): + async def get_stacks(self, region: str): - stacks = await AWSFacadeUtils.get_all_pages('cloudformation', region, self.session, 'list_stacks', 'StackSummaries') + stacks = await AWSFacadeUtils.get_all_pages( + 'cloudformation', region, self.session, 'list_stacks', 'StackSummaries') stacks = [stack for stack in stacks if not CloudFormation._is_stack_deleted(stack)] - client = AWSFacadeUtils.get_client('cloudformation', self.session, region) - for stack in stacks: - stack_name = stack['StackName'] - - stack_description = await run_concurrently( - lambda: client.describe_stacks(StackName=stack_name)['Stacks'][0] - ) - stack.update(stack_description) - stack['template'] = await run_concurrently( - lambda: client.get_template(StackName=stack_name)['TemplateBody'] - ) - stack_policy = await run_concurrently( - lambda: client.get_stack_policy(StackName=stack_name) - ) - if 'StackPolicyBody' in stack_policy: - stack['policy'] = json.loads(stack_policy['StackPolicyBody']) + await AWSFacadeUtils.get_and_set_concurrently( + [self._get_and_set_description, self._get_and_set_template, self._get_and_set_policy], + stacks, region=region) return stacks + async def _get_and_set_description(self, stack: {}, region: str): + client = AWSFacadeUtils.get_client('cloudformation', self.session, region) + stack_description = await run_concurrently( + lambda: client.describe_stacks(StackName=stack['StackName'])['Stacks'][0]) + stack.update(stack_description) + + async def _get_and_set_template(self, stack: {}, region: str): + client = AWSFacadeUtils.get_client('cloudformation', self.session, region) + stack['template'] = await run_concurrently( + lambda: client.get_template(StackName=stack['StackName'])['TemplateBody']) + + async def _get_and_set_policy(self, stack: {}, region: str): + client = AWSFacadeUtils.get_client('cloudformation', self.session, region) + stack_policy = await run_concurrently( + lambda: client.get_stack_policy(StackName=stack['StackName'])) + if 'StackPolicyBody' in stack_policy: + stack['policy'] = json.loads(stack_policy['StackPolicyBody']) + @staticmethod def _is_stack_deleted(stack): - return stack.get('StackStatus', None) == 'DELETE_COMPLETE' \ No newline at end of file + return stack.get('StackStatus', None) == 'DELETE_COMPLETE' From 06523b5b35e98efea14065a7159dc72b3065e194 Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 30 Mar 2019 15:15:05 -0400 Subject: [PATCH 644/667] Improve concurrent support for CloudTrail facade. --- ScoutSuite/providers/aws/facade/cloudtrail.py | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/cloudtrail.py b/ScoutSuite/providers/aws/facade/cloudtrail.py index 78ba4cbc3..6f41cc0cb 100644 --- a/ScoutSuite/providers/aws/facade/cloudtrail.py +++ b/ScoutSuite/providers/aws/facade/cloudtrail.py @@ -7,15 +7,19 @@ class CloudTrailFacade(AWSBaseFacade): async def get_trails(self, region): client = AWSFacadeUtils.get_client('cloudtrail', self.session, region) trails = await run_concurrently( - lambda: client.describe_trails()['trailList'] - ) - - for trail in trails: - trail.update(await run_concurrently( - lambda: client.get_trail_status(Name=trail['TrailARN']) - )) - trail['EventSelectors'] = await run_concurrently( - lambda: client.get_event_selectors(TrailName=trail['TrailARN'])['EventSelectors'] - ) + lambda: client.describe_trails()['trailList']) + await AWSFacadeUtils.get_and_set_concurrently( + [self._get_and_set_status, self._get_and_set_selectors], trails, region=region) return trails + + async def _get_and_set_status(self, trail: {}, region: str): + client = AWSFacadeUtils.get_client('cloudtrail', self.session, region) + trail_status = await run_concurrently( + lambda: client.get_trail_status(Name=trail['TrailARN'])) + trail.update(trail_status) + + async def _get_and_set_selectors(self, trail: {}, region: str): + client = AWSFacadeUtils.get_client('cloudtrail', self.session, region) + trail['EventSelectors'] = await run_concurrently( + lambda: client.get_event_selectors(TrailName=trail['TrailARN'])['EventSelectors']) From 79535252d95506a3024641810320622e2d94a36f Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 30 Mar 2019 15:15:19 -0400 Subject: [PATCH 645/667] Improve concurrent support for ec2 facade. --- ScoutSuite/providers/aws/facade/ec2.py | 27 +++++++++----------------- 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/ec2.py b/ScoutSuite/providers/aws/facade/ec2.py index 288cefeec..852494873 100644 --- a/ScoutSuite/providers/aws/facade/ec2.py +++ b/ScoutSuite/providers/aws/facade/ec2.py @@ -46,9 +46,7 @@ async def get_vpcs(self, region: str): async def get_images(self, region: str, owner_id: str): filters = [{'Name': 'owner-id', 'Values': [owner_id]}] client = AWSFacadeUtils.get_client('ec2', self.session, region) - response = await run_concurrently(lambda: client.describe_images(Filters=filters)) - - return response['Images'] + return await run_concurrently(lambda: client.describe_images(Filters=filters)['Images']) async def get_network_interfaces(self, region: str, vpc: str): filters = [{'Name': 'vpc-id', 'Values': [vpc]}] @@ -62,15 +60,16 @@ async def get_snapshots(self, region: str, owner_id: str): filters = [{'Name': 'owner-id', 'Values': [owner_id]}] snapshots = await AWSFacadeUtils.get_all_pages( 'ec2', region, self.session, 'describe_snapshots', 'Snapshots', Filters=filters) + await AWSFacadeUtils.get_and_set_concurrently([self._get_and_set_snapshot_attributes], snapshots, region=region) + + return snapshots + async def _get_and_set_snapshot_attributes(self, snapshot: {}, region: str): ec2_client = AWSFacadeUtils.get_client('ec2', self.session, region) - for snapshot in snapshots: - snapshot['CreateVolumePermissions'] = await run_concurrently(lambda: ec2_client.describe_snapshot_attribute( + snapshot['CreateVolumePermissions'] = await run_concurrently(lambda: ec2_client.describe_snapshot_attribute( Attribute='createVolumePermission', SnapshotId=snapshot['SnapshotId'])['CreateVolumePermissions']) - return snapshots - async def get_network_acls(self, region: str, vpc: str): filters = [{'Name': 'vpc-id', 'Values': [vpc]}] return await AWSFacadeUtils.get_all_pages( @@ -91,20 +90,12 @@ async def cache_flow_logs(self, region: str): async def get_subnets(self, region: str, vpc: str): ec2_client = AWSFacadeUtils.get_client('ec2', self.session, region) filters = [{'Name': 'vpc-id', 'Values': [vpc]}] - subnets = await run_concurrently( - lambda: ec2_client.describe_subnets(Filters=filters)['Subnets'] - ) - # Fetch and set subnet flow logs concurrently: - tasks = { - asyncio.ensure_future( - self.get_and_set_subnet_flow_logs(region, subnet) - ) for subnet in subnets - } - await asyncio.wait(tasks) + subnets = await run_concurrently(lambda: ec2_client.describe_subnets(Filters=filters)['Subnets']) + await AWSFacadeUtils.get_and_set_concurrently([self._get_and_set_subnet_flow_logs], subnets, region=region) return subnets - async def get_and_set_subnet_flow_logs(self, region: str, subnet: {}): + async def _get_and_set_subnet_flow_logs(self, subnet: {}, region: str): await self.cache_flow_logs(region) subnet['flow_logs'] =\ [flow_log for flow_log in self.flow_logs_cache[region] From 2df9aeac119d0775ecee5757cdce20bb86bc5176 Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 30 Mar 2019 15:15:33 -0400 Subject: [PATCH 646/667] Improve concurrent support for EFS facade. --- ScoutSuite/providers/aws/facade/efs.py | 47 +++++++++++++++----------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/efs.py b/ScoutSuite/providers/aws/facade/efs.py index 9db95dcbc..8ce9f500f 100644 --- a/ScoutSuite/providers/aws/facade/efs.py +++ b/ScoutSuite/providers/aws/facade/efs.py @@ -7,26 +7,33 @@ class EFSFacade(AWSBaseFacade): async def get_file_systems(self, region: str): file_systems = await AWSFacadeUtils.get_all_pages( 'efs', region, self.session, 'describe_file_systems', 'FileSystems') + await AWSFacadeUtils.get_and_set_concurrently( + [self._get_and_set_tags, self._get_and_set_mount_targets], file_systems, region=region) + return file_systems + + async def _get_and_set_tags(self, file_system: {}, region: str): client = AWSFacadeUtils.get_client('efs', self.session, region) - for file_system in file_systems: - file_system_id = file_system['FileSystemId'] - file_system['Tags'] = await run_concurrently( - lambda: client.describe_tags(FileSystemId=file_system_id)['Tags']) - - # Get mount targets - mount_targets = await AWSFacadeUtils.get_all_pages( - 'efs', region, self.session, 'describe_mount_targets', 'MountTargets', FileSystemId=file_system_id) - file_system['MountTargets'] = {} - for mount_target in mount_targets: - mount_target_id = mount_target['MountTargetId'] - file_system['MountTargets'][mount_target_id] = mount_target - - # Get security groups - security_groups = run_concurrently( - lambda: client.describe_mount_target_security_groups( - MountTargetId=mount_target_id)['SecurityGroups'] - ) - file_system['MountTargets'][mount_target_id]['SecurityGroups'] = security_groups + file_system['Tags'] = await run_concurrently( + lambda: client.describe_tags(FileSystemId=file_system['FileSystemId'])['Tags']) - return file_systems + async def _get_and_set_mount_targets(self, file_system: {}, region: str): + file_system['MountTargets'] = {} + mount_targets = await AWSFacadeUtils.get_all_pages( + 'efs', region, self.session, 'describe_mount_targets', 'MountTargets', + FileSystemId=file_system['FileSystemId']) + + if len(mount_targets) == 0: + return + + for mount_target in mount_targets: + mount_target_id = mount_target['MountTargetId'] + file_system['MountTargets'][mount_target_id] = mount_target + + await AWSFacadeUtils.get_and_set_concurrently( + [self._get_and_set_mount_target_security_groups], mount_targets, region=region) + + async def _get_and_set_mount_target_security_groups(self, mount_target: {}, region: str): + client = AWSFacadeUtils.get_client('efs', self.session, region) + mount_target['SecurityGroups'] = run_concurrently(lambda: client.describe_mount_target_security_groups( + MountTargetId=mount_target['MountTargetId'])['SecurityGroups']) From 4dd95765963a7d91471db569d928bd4f3e99cf7a Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 30 Mar 2019 15:15:54 -0400 Subject: [PATCH 647/667] Improve concurrent support for Elasticache facade. --- .../providers/aws/facade/elasticache.py | 43 +++++++++---------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/elasticache.py b/ScoutSuite/providers/aws/facade/elasticache.py index d7a6bb2fb..a1bd9f6cf 100644 --- a/ScoutSuite/providers/aws/facade/elasticache.py +++ b/ScoutSuite/providers/aws/facade/elasticache.py @@ -1,10 +1,9 @@ -import boto3 - from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade from ScoutSuite.providers.aws.utils import ec2_classic from asyncio import Lock + class ElastiCacheFacade(AWSBaseFacade): regional_clusters_cache_locks = {} regional_subnets_cache_locks = {} @@ -20,20 +19,28 @@ async def cache_clusters(self, region): if region in self.clusters_cache: return - self.clusters_cache[region] = await AWSFacadeUtils.get_all_pages('elasticache', region, self.session, 'describe_cache_clusters', 'CacheClusters') + self.clusters_cache[region] = await AWSFacadeUtils.get_all_pages( + 'elasticache', region, self.session, 'describe_cache_clusters', 'CacheClusters') - for cluster in self.clusters_cache[region]: - if 'CacheSubnetGroupName' not in cluster: - cluster['VpcId'] = ec2_classic - else: - subnet_group = await self.get_subnet_group(region, cluster['CacheSubnetGroupName']) - cluster['VpcId'] = subnet_group['VpcId'] + await AWSFacadeUtils.get_and_set_concurrently( + [self._get_and_set_cluster_vpc], self.clusters_cache[region], region=region) + + async def _get_and_set_cluster_vpc(self, cluster: {}, region: str): + if 'CacheSubnetGroupName' not in cluster: + cluster['VpcId'] = ec2_classic + else: + subnets = await AWSFacadeUtils.get_all_pages( + 'elasticache', region, self.session, 'describe_cache_subnet_groups', 'CacheSubnetGroups', + CacheSubnetGroupName=cluster['CacheSubnetGroupName']) + subnet_group = subnets[0] + cluster['VpcId'] = subnet_group['VpcId'] async def get_security_groups(self, region): client = AWSFacadeUtils.get_client('elasticache', self.session, region) try: - return await AWSFacadeUtils.get_all_pages('elasticache', region, self.session, 'describe_cache_security_groups', 'CacheSecurityGroups') + return await AWSFacadeUtils.get_all_pages( + 'elasticache', region, self.session, 'describe_cache_security_groups', 'CacheSecurityGroups') except client.exceptions.InvalidParameterValueException: # Recent account are not allowed to use security groups at this level. Calling @@ -46,22 +53,14 @@ async def get_subnet_groups(self, region, vpc): await self.cache_subnets(region) return [subnet for subnet in self.subnets_cache[region] if subnet['VpcId'] == vpc] - async def get_subnet_group(self, region, subnet_name): - subnets = await AWSFacadeUtils.get_all_pages('elasticache', \ - region, \ - self.session, \ - 'describe_cache_subnet_groups', \ - 'CacheSubnetGroups', \ - CacheSubnetGroupName=subnet_name \ - ) - return subnets[0] - async def cache_subnets(self, region): async with self.regional_subnets_cache_locks.setdefault(region, Lock()): if region in self.subnets_cache: return - self.subnets_cache[region] = await AWSFacadeUtils.get_all_pages('elasticache', region, self.session, 'describe_cache_subnet_groups', 'CacheSubnetGroups') + self.subnets_cache[region] = await AWSFacadeUtils.get_all_pages( + 'elasticache', region, self.session, 'describe_cache_subnet_groups', 'CacheSubnetGroups') async def get_parameter_groups(self, region): - return await AWSFacadeUtils.get_all_pages('elasticache', region, self.session, 'describe_cache_parameter_groups', 'CacheParameterGroups') \ No newline at end of file + return await AWSFacadeUtils.get_all_pages( + 'elasticache', region, self.session, 'describe_cache_parameter_groups', 'CacheParameterGroups') From 4ff576936bb6be58b632409d32eb5dd030dc891e Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 30 Mar 2019 15:16:10 -0400 Subject: [PATCH 648/667] Improve concurrent support for ELB facade. --- ScoutSuite/providers/aws/facade/elb.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/elb.py b/ScoutSuite/providers/aws/facade/elb.py index 71eb7b4e9..179c75651 100644 --- a/ScoutSuite/providers/aws/facade/elb.py +++ b/ScoutSuite/providers/aws/facade/elb.py @@ -27,16 +27,10 @@ async def cache_load_balancers(self, region): load_balancer['VpcId'] =\ load_balancer['VPCId'] if 'VPCId' in load_balancer and load_balancer['VPCId'] else ec2_classic - if len(self.load_balancers_cache[region]) > 0: - # Fetch and set the attributes of all the load balancers concurrently: - tasks = { - asyncio.ensure_future( - self._get_and_set_load_balancer_attributes(region, load_balancer) - ) for load_balancer in self.load_balancers_cache[region] - } - await asyncio.wait(tasks) - - async def _get_and_set_load_balancer_attributes(self, region: str, load_balancer: {}): + await AWSFacadeUtils.get_and_set_concurrently( + [self._get_and_set_load_balancer_attributes], self.load_balancers_cache[region], region=region) + + async def _get_and_set_load_balancer_attributes(self, load_balancer: {}, region: str): elb_client = AWSFacadeUtils.get_client('elb', self.session, region) load_balancer['attributes'] = await run_concurrently( lambda: elb_client.describe_load_balancer_attributes( From ca44cb1f04ef096e03269e0c5948e29af8de7add Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 30 Mar 2019 15:16:24 -0400 Subject: [PATCH 649/667] Improve concurrent support for ELBv2 facade. --- ScoutSuite/providers/aws/facade/elbv2.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/elbv2.py b/ScoutSuite/providers/aws/facade/elbv2.py index 62538bcfc..8368dab16 100644 --- a/ScoutSuite/providers/aws/facade/elbv2.py +++ b/ScoutSuite/providers/aws/facade/elbv2.py @@ -27,16 +27,10 @@ async def cache_load_balancers(self, region): load_balancer['VpcId'] =\ load_balancer['VpcId'] if 'VpcId' in load_balancer and load_balancer['VpcId'] else ec2_classic - if len(self.load_balancers_cache[region]) == 0: - return - tasks = { - asyncio.ensure_future( - self.get_and_set_load_balancer_attributes(region, load_balancer) - ) for load_balancer in self.load_balancers_cache[region] - } - await asyncio.wait(tasks) - - async def get_and_set_load_balancer_attributes(self, region: str, load_balancer: dict): + await AWSFacadeUtils.get_and_set_concurrently( + [self._get_and_set_load_balancer_attributes], self.load_balancers_cache[region], region=region) + + async def _get_and_set_load_balancer_attributes(self, load_balancer: dict, region: str): elbv2_client = AWSFacadeUtils.get_client('elbv2', self.session, region) load_balancer['attributes'] = await run_concurrently( lambda: elbv2_client.describe_load_balancer_attributes( From ecbed0ee14c5d905640540e4bf6bcf6e1d85954a Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 30 Mar 2019 15:17:00 -0400 Subject: [PATCH 650/667] Use map_concurrently function instead of run_tasks_concurrently. --- ScoutSuite/providers/aws/facade/emr.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/emr.py b/ScoutSuite/providers/aws/facade/emr.py index 7acadced6..890be55ba 100644 --- a/ScoutSuite/providers/aws/facade/emr.py +++ b/ScoutSuite/providers/aws/facade/emr.py @@ -1,15 +1,15 @@ from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade -from ScoutSuite.providers.utils import run_concurrently, run_tasks_concurrently -import asyncio +from ScoutSuite.providers.utils import map_concurrently, run_concurrently class EMRFacade(AWSBaseFacade): async def get_clusters(self, region): - clusters_list = await AWSFacadeUtils.get_all_pages('emr', region, self.session, 'list_clusters', 'Clusters') + cluster_list = await AWSFacadeUtils.get_all_pages('emr', region, self.session, 'list_clusters', 'Clusters') + cluster_ids = [cluster['Id'] for cluster in cluster_list] client = AWSFacadeUtils.get_client('emr', self.session, region) - cluster_ids = [cluster['Id'] for cluster in clusters_list] - - return await run_tasks_concurrently({ - run_concurrently(lambda: client.describe_cluster(ClusterId=id)['Cluster']) for id in cluster_ids - }) + + return await map_concurrently( + lambda cluster_id: run_concurrently( + lambda: client.describe_cluster(ClusterId=cluster_id)['Cluster']), + cluster_ids) From ade1c8e0ef19f57bc8386eae33ed0b46313a5601 Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 30 Mar 2019 15:17:12 -0400 Subject: [PATCH 651/667] Improve concurrent support for IAM facade. --- ScoutSuite/providers/aws/facade/iam.py | 190 +++++++++++-------------- 1 file changed, 84 insertions(+), 106 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/iam.py b/ScoutSuite/providers/aws/facade/iam.py index 7b1cbd2e6..29db39a98 100644 --- a/ScoutSuite/providers/aws/facade/iam.py +++ b/ScoutSuite/providers/aws/facade/iam.py @@ -1,9 +1,10 @@ +import asyncio +import functools + from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade -from ScoutSuite.core.console import print_error, print_exception -from ScoutSuite.providers.utils import run_concurrently, get_non_provider_id -from ScoutSuite.providers.aws.utils import is_throttled -from ScoutSuite.providers.utils import run_concurrently, run_tasks_concurrently +from ScoutSuite.core.console import print_error +from ScoutSuite.providers.utils import get_non_provider_id, run_concurrently class IAMFacade(AWSBaseFacade): @@ -35,28 +36,27 @@ async def get_credential_reports(self): async def get_groups(self): groups = await AWSFacadeUtils.get_all_pages('iam', None, self.session, 'list_groups', 'Groups') - for group in groups: - group['Users'] = await self._fetch_group_users(group['GroupName']) - policies = await self._get_inline_policies( - 'group', group['GroupId'], group['GroupName']) - if len(policies): - group['inline_policies'] = policies - group['inline_policies_count'] = len(policies) + await AWSFacadeUtils.get_and_set_concurrently( + [self._get_and_set_group_users, + functools.partial(self._get_and_set_inline_policies, iam_resource_type='group')], groups) return groups async def get_policies(self): - policies = await AWSFacadeUtils.get_all_pages('iam', None, self.session, 'list_policies', 'Policies', OnlyAttached=True) - return await run_tasks_concurrently({ - self.get_detailed_policy(policy) for policy in policies - }) + policies = await AWSFacadeUtils.get_all_pages( + 'iam', None, self.session, 'list_policies', 'Policies', OnlyAttached=True) + await AWSFacadeUtils.get_and_set_concurrently([self._get_and_set_policy_details], policies) + return policies - async def get_detailed_policy(self, policy): + async def _get_and_set_policy_details(self, policy): client = AWSFacadeUtils.get_client('iam', self.session) - policy_version = await run_concurrently(lambda: client.get_policy_version(PolicyArn=policy['Arn'], VersionId=policy['DefaultVersionId'])) + policy_version = await run_concurrently( + lambda: client.get_policy_version(PolicyArn=policy['Arn'], VersionId=policy['DefaultVersionId'])) policy['PolicyDocument'] = policy_version['PolicyVersion']['Document'] policy['attached_to'] = {} - attached_entities = await AWSFacadeUtils.get_multiple_entities_from_all_pages('iam', None, self.session, 'list_entities_for_policy', ['PolicyGroups', 'PolicyRoles', 'PolicyUsers'], PolicyArn=policy['Arn']) + attached_entities = await AWSFacadeUtils.get_multiple_entities_from_all_pages( + 'iam', None, self.session, 'list_entities_for_policy', ['PolicyGroups', 'PolicyRoles', 'PolicyUsers'], + PolicyArn=policy['Arn']) for entity_type in attached_entities: resource_type = entity_type.replace('Policy', '').lower() @@ -72,127 +72,105 @@ async def get_detailed_policy(self, policy): policy['attached_to'][resource_type].append( {'name': resource_name, 'id': resource_id}) - return policy - async def get_users(self): users = await AWSFacadeUtils.get_all_pages('iam', None, self.session, 'list_users', 'Users') - return await run_tasks_concurrently({ - self.get_detailed_user(user) for user in users - }) + await AWSFacadeUtils.get_and_set_concurrently( + [functools.partial(self._get_and_set_inline_policies, iam_resource_type='user'), + self._get_and_set_user_groups, + self._get_and_set_user_login_profile, + self._get_and_set_user_acces_keys, + self._get_and_set_user_mfa_devices], + users) + return users - async def get_detailed_user(self, user): + async def _get_and_set_user_login_profile(self, user: {}): client = AWSFacadeUtils.get_client('iam', self.session) - user_name = user['UserName'] - user_id = user['UserId'] - - policies = await self._get_inline_policies('user', user_id, user_name) - if len(policies): - user['inline_policies'] = policies - user['inline_policies_count'] = len(policies) - user['groups'] = [] - groups = await AWSFacadeUtils.get_all_pages('iam', None, self.session, 'list_groups_for_user', 'Groups', UserName=user_name) - for group in groups: - user['groups'].append(group['GroupName']) try: user['LoginProfile'] = await run_concurrently( - lambda: client.get_login_profile(UserName=user_name)['LoginProfile']) + lambda: client.get_login_profile(UserName=user['UserName'])['LoginProfile']) except Exception: pass - user['AccessKeys'] = await self._get_user_acces_keys(user_name) - user['MFADevices'] = await self._get_user_mfa_devices(user_name) - return user + async def _get_and_set_user_groups(self, user: {}): + groups = await AWSFacadeUtils.get_all_pages( + 'iam', None, self.session, 'list_groups_for_user', 'Groups',UserName=user['UserName']) + user['groups'] = [group['GroupName'] for group in groups] async def get_roles(self): roles = await AWSFacadeUtils.get_all_pages('iam', None, self.session, 'list_roles', 'Roles') - - # Handle throttling errors for role in roles: role['instances_count'] = 'N/A' - - # Get role policies - policies = await self._get_inline_policies( - 'role', role['RoleId'], role['RoleName']) - if len(policies): - role['inline_policies'] = policies - role['inline_policies_count'] = len(policies) - - # Get instance profiles - profiles = await AWSFacadeUtils.get_all_pages('iam', None, self.session, 'list_instance_profiles_for_role', 'InstanceProfiles', RoleName=role['RoleName']) - role.setdefault('instance_profiles', {}) - for profile in profiles: - profile_id = profile['InstanceProfileId'] - role['instance_profiles'].setdefault(profile_id, {}) - role['instance_profiles'][profile_id].setdefault( - 'arn', profile['Arn']) - role['instance_profiles'][profile_id].setdefault( - 'name', profile['InstanceProfileName']) - # Get trust relationship role['assume_role_policy'] = {} role['assume_role_policy']['PolicyDocument'] = role.pop( 'AssumeRolePolicyDocument') + await AWSFacadeUtils.get_and_set_concurrently( + [functools.partial(self._get_and_set_inline_policies, iam_resource_type='role'), + self._get_and_set_role_profiles], roles) return roles + async def _get_and_set_role_profiles(self, role: {}): + profiles = await AWSFacadeUtils.get_all_pages( + 'iam', None, self.session, 'list_instance_profiles_for_role', 'InstanceProfiles', + RoleName=role['RoleName']) + role.setdefault('instance_profiles', {}) + for profile in profiles: + profile_id = profile['InstanceProfileId'] + role['instance_profiles'].setdefault(profile_id, {}) + role['instance_profiles'][profile_id].setdefault( + 'arn', profile['Arn']) + role['instance_profiles'][profile_id].setdefault( + 'name', profile['InstanceProfileName']) + async def get_password_policy(self): client = AWSFacadeUtils.get_client('iam', self.session) return (await run_concurrently(client.get_account_password_policy))['PasswordPolicy'] - async def _get_user_acces_keys(self, user_name): + async def _get_and_set_user_acces_keys(self, user: {}): client = AWSFacadeUtils.get_client('iam', self.session) - response = await run_concurrently(lambda: client.list_access_keys(UserName=user_name)) - return response['AccessKeyMetadata'] + user['AccessKeys'] = await run_concurrently( + lambda: client.list_access_keys(UserName=user['UserName'])['AccessKeyMetadata']) - async def _get_user_mfa_devices(self, user_name): + async def _get_and_set_user_mfa_devices(self, user: {}): client = AWSFacadeUtils.get_client('iam', self.session) - response = await run_concurrently(lambda: client.list_mfa_devices(UserName=user_name)) - return response['MFADevices'] + user['MFADevices'] = await run_concurrently( + lambda: client.list_mfa_devices(UserName=user['UserName'])['MFADevices']) - async def _fetch_group_users(self, group_name): + async def _get_and_set_group_users(self, group: {}): client = AWSFacadeUtils.get_client('iam', self.session) - fetched_users = client.get_group(GroupName=group_name)['Users'] - - users = [] - for user in fetched_users: - users.append(user['UserId']) - return users + users = await run_concurrently(lambda: client.get_group(GroupName=group['GroupName'])['Users']) + group['Users'] = [user['UserId'] for user in users] - async def _get_inline_policies(self, iam_resource_type, resource_id, resource_name): + async def _get_and_set_inline_policies(self, resource, iam_resource_type): client = AWSFacadeUtils.get_client('iam', self.session) - get_policy_method = getattr( - client, 'get_' + iam_resource_type + '_policy') - fetched_policies = {} - list_policy_method = getattr( - client, 'list_' + iam_resource_type + '_policies') + list_policy_method = getattr(client, 'list_' + iam_resource_type + '_policies') + resource_name = resource[iam_resource_type.title() + 'Name'] args = {iam_resource_type.title() + 'Name': resource_name} - - try: - policy_names = await run_concurrently(lambda: list_policy_method(**args)['PolicyNames']) - except Exception as e: - if is_throttled(e): - raise e - else: - print_exception(e) - return fetched_policies - - try: - for policy_name in policy_names: - args['PolicyName'] = policy_name - policy_document = get_policy_method(**args)['PolicyDocument'] - policy_id = get_non_provider_id(policy_name) - fetched_policies[policy_id] = {} - fetched_policies[policy_id]['PolicyDocument'] = self._normalize_statements( - policy_document) - fetched_policies[policy_id]['name'] = policy_name - fetched_policies[policy_id] = fetched_policies[policy_id] - except Exception as e: - if is_throttled(e): - raise e - else: - print_exception(e) - - return fetched_policies + + resource['inline_policies'] = {} + policy_names = await run_concurrently(lambda: list_policy_method(**args)['PolicyNames']) + if len(policy_names) == 0: + resource['inline_policies_count'] = 0 + return + + get_policy_method = getattr(client, 'get_' + iam_resource_type + '_policy') + tasks = { + asyncio.ensure_future( + run_concurrently(lambda: get_policy_method(**dict(args, PolicyName=policy_name))) + ) for policy_name in policy_names + } + for task in asyncio.as_completed(tasks): + policy = await task + policy_name = policy['PolicyName'] + policy_id = get_non_provider_id(policy_name) + policy_document = policy['PolicyDocument'] + + resource['inline_policies'][policy_id] = {} + resource['inline_policies'][policy_id]['PolicyDocument'] = self._normalize_statements( + policy_document) + resource['inline_policies'][policy_id]['name'] = policy_name + resource['inline_policies_count'] = len(resource['inline_policies']) def _normalize_statements(self, policy_document): for statement in policy_document['Statement']: From 3e1fdda96ef689205ed2f9cfd1243a1cdc84356f Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 30 Mar 2019 15:17:46 -0400 Subject: [PATCH 652/667] Improve concurrent support for RDS facade. --- ScoutSuite/providers/aws/facade/rds.py | 81 ++++++++++++++++---------- 1 file changed, 51 insertions(+), 30 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/rds.py b/ScoutSuite/providers/aws/facade/rds.py index 7c93fd7d8..9fdaf9931 100644 --- a/ScoutSuite/providers/aws/facade/rds.py +++ b/ScoutSuite/providers/aws/facade/rds.py @@ -24,20 +24,26 @@ async def _cache_instances(self, region: str): if region in self._instances_cache: return - self._instances_cache[region] = await AWSFacadeUtils.get_all_pages('rds', region, self.session, 'describe_db_instances', 'DBInstances') + self._instances_cache[region] = await AWSFacadeUtils.get_all_pages( + 'rds', region, self.session, 'describe_db_instances', 'DBInstances') - client = AWSFacadeUtils.get_client('rds', self.session, region) for instance in self._instances_cache[region]: instance['VpcId'] = instance['DBSubnetGroup']['VpcId'] \ if 'DBSubnetGroup' in instance and 'VpcId' in instance['DBSubnetGroup'] \ and instance['DBSubnetGroup']['VpcId'] \ else ec2_classic - if 'DBClusterIdentifier' in instance: - cluster_id = instance['DBClusterIdentifier'] - clusters = await run_concurrently(lambda: client.describe_db_clusters(DBClusterIdentifier=cluster_id)) - cluster = clusters['DBClusters'][0] - instance['MultiAZ'] = cluster['MultiAZ'] + await AWSFacadeUtils.get_and_set_concurrently( + [self._get_and_set_instance_clusters], self._instances_cache[region], region=region) + + async def _get_and_set_instance_clusters(self, instance: {}, region: str): + client = AWSFacadeUtils.get_client('rds', self.session, region) + if 'DBClusterIdentifier' in instance: + cluster_id = instance['DBClusterIdentifier'] + clusters = await run_concurrently( + lambda: client.describe_db_clusters(DBClusterIdentifier=cluster_id)) + cluster = clusters['DBClusters'][0] + instance['MultiAZ'] = cluster['MultiAZ'] async def get_snapshots(self, region: str, vpc: str): await self._cache_snapshots(region) @@ -48,14 +54,22 @@ async def _cache_snapshots(self, region: str): if region in self._snapshots_cache: return - client = AWSFacadeUtils.get_client('rds', self.session, region) - self._snapshots_cache[region] = await AWSFacadeUtils.get_all_pages('rds', region, self.session, 'describe_db_snapshots', 'DBSnapshots') + self._snapshots_cache[region] = await AWSFacadeUtils.get_all_pages( + 'rds', region, self.session, 'describe_db_snapshots', 'DBSnapshots') + for snapshot in self._snapshots_cache[region]: - snapshot_id = snapshot['DBSnapshotIdentifier'] snapshot['VpcId'] = snapshot['VpcId'] if 'VpcId' in snapshot else ec2_classic - attributes_response = await run_concurrently(lambda: client.describe_db_snapshot_attributes(DBSnapshotIdentifier=snapshot_id)) - attributes = attributes_response['DBSnapshotAttributesResult'] - snapshot['Attributes'] = attributes['DBSnapshotAttributes'] if 'DBSnapshotAttributes' in attributes else {} + + await AWSFacadeUtils.get_and_set_concurrently( + [self._get_and_set_snapshot_attributes], self._snapshots_cache[region], region=region) + + async def _get_and_set_snapshot_attributes(self, snapshot: {}, region: str): + client = AWSFacadeUtils.get_client('rds', self.session, region) + attributes = await run_concurrently( + lambda: client.describe_db_snapshot_attributes( + DBSnapshotIdentifier=snapshot['DBSnapshotIdentifier'])['DBSnapshotAttributesResult']) + snapshot['Attributes'] =\ + attributes['DBSnapshotAttributes'] if 'DBSnapshotAttributes' in attributes else {} async def get_subnet_groups(self, region: str, vpc: str): await self._cache_subnet_groups(region) @@ -66,26 +80,33 @@ async def _cache_subnet_groups(self, region: str): if region in self._subnet_groups_cache: return - self._subnet_groups_cache[region] = await AWSFacadeUtils.get_all_pages('rds', region, self.session, 'describe_db_subnet_groups', 'DBSubnetGroups') + self._subnet_groups_cache[region] = await AWSFacadeUtils.get_all_pages( + 'rds', region, self.session, 'describe_db_subnet_groups', 'DBSubnetGroups') async def get_parameter_groups(self, region: str): - parameter_groups = await AWSFacadeUtils.get_all_pages('rds', region, self.session, 'describe_db_parameter_groups', 'DBParameterGroups') - for parameter_group in parameter_groups: - name = parameter_group['DBParameterGroupName'] - try: - parameters = await AWSFacadeUtils.get_all_pages('rds', region, self.session, 'describe_db_parameters', 'Parameters', DBParameterGroupName=name) - parameter_group['Parameters'] = {} - for parameter in parameters: - # Discard non-modifiable parameters - if not parameter['IsModifiable']: - continue - parameter_name = parameter.pop('ParameterName') - parameter_group['Parameters'][parameter_name] = parameter - except Exception as e: - print_exception(e) - print_error('Failed fetching DB parameters for %s' % name) + parameter_groups = await AWSFacadeUtils.get_all_pages( + 'rds', region, self.session, 'describe_db_parameter_groups', 'DBParameterGroups') + await AWSFacadeUtils.get_and_set_concurrently( + [self._get_and_set_db_parameters], parameter_groups, region=region) return parameter_groups + async def _get_and_set_db_parameters(self, parameter_group: {}, region: str): + name = parameter_group['DBParameterGroupName'] + try: + parameters = await AWSFacadeUtils.get_all_pages( + 'rds', region, self.session, 'describe_db_parameters', 'Parameters', DBParameterGroupName=name) + parameter_group['Parameters'] = {} + for parameter in parameters: + # Discard non-modifiable parameters + if not parameter['IsModifiable']: + continue + parameter_name = parameter.pop('ParameterName') + parameter_group['Parameters'][parameter_name] = parameter + except Exception as e: + print_exception(e) + print_error('Failed fetching DB parameters for %s' % name) + async def get_security_groups(self, region: str) : - return await AWSFacadeUtils.get_all_pages('rds', region, self.session, 'describe_db_security_groups', 'DBSecurityGroups') + return await AWSFacadeUtils.get_all_pages( + 'rds', region, self.session, 'describe_db_security_groups', 'DBSecurityGroups') From d10f52a20455ce1b26760d0f909d04c5f0cf3956 Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 30 Mar 2019 15:18:05 -0400 Subject: [PATCH 653/667] Improve concurrent support for Redshift facade. --- ScoutSuite/providers/aws/facade/redshift.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/redshift.py b/ScoutSuite/providers/aws/facade/redshift.py index 05f4d42de..3ffa0c84b 100644 --- a/ScoutSuite/providers/aws/facade/redshift.py +++ b/ScoutSuite/providers/aws/facade/redshift.py @@ -19,9 +19,8 @@ async def cache_clusters(self, region): if region in self.clusters_cache: return - self.clusters_cache[region] =\ - await AWSFacadeUtils.get_all_pages('redshift', region, self.session, - 'describe_clusters', 'Clusters') + self.clusters_cache[region] = await AWSFacadeUtils.get_all_pages( + 'redshift', region, self.session, 'describe_clusters', 'Clusters') for cluster in self.clusters_cache[region]: cluster['VpcId'] =\ From dd9e52fdfde3bc1e7ddfde7db4dbe110c13d7745 Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 30 Mar 2019 15:18:23 -0400 Subject: [PATCH 654/667] Improve concurrent support for S3 facade. --- ScoutSuite/providers/aws/facade/s3.py | 64 +++++++++++++-------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/s3.py b/ScoutSuite/providers/aws/facade/s3.py index 4e0728cc4..5e6b99a4c 100644 --- a/ScoutSuite/providers/aws/facade/s3.py +++ b/ScoutSuite/providers/aws/facade/s3.py @@ -1,45 +1,48 @@ -from asyncio import Lock import json from botocore.exceptions import ClientError from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade from ScoutSuite.providers.utils import run_concurrently -from ScoutSuite.providers.utils import get_non_provider_id -from ScoutSuite.core.console import print_error, print_exception +from ScoutSuite.core.console import print_error class S3Facade(AWSBaseFacade): - async def get_buckets(self): client = AWSFacadeUtils.get_client('s3', self.session) - buckets = (await run_concurrently(client.list_buckets))['Buckets'] - + buckets = await run_concurrently(lambda: client.list_buckets()['Buckets']) + + # We need first to retrieve bucket locations before retrieving bucket details: + await AWSFacadeUtils.get_and_set_concurrently([self._get_and_set_s3_bucket_location], buckets) + + # Then we can retrieve bucket details concurrently: + await AWSFacadeUtils.get_and_set_concurrently( + [self._get_and_set_s3_bucket_logging, + self._get_and_set_s3_bucket_versioning, + self._get_and_set_s3_bucket_webhosting, + self._get_and_set_s3_bucket_default_encryption, + self._get_and_set_s3_acls, + self._get_and_set_s3_bucket_policy], + buckets) + + # Non-async post-processing: for bucket in buckets: - bucket_name = bucket['Name'] - bucket['region'] = await self._get_s3_bucket_location(bucket_name) - await self._set_s3_bucket_logging(bucket) - await self._set_s3_bucket_versioning(bucket) - await self._set_s3_bucket_webhosting(bucket) - await self._set_s3_bucket_default_encryption(bucket) - await self._set_s3_acls(bucket) - await self._set_s3_bucket_policy(bucket) self._set_s3_bucket_secure_transport(bucket) return buckets - async def _get_s3_bucket_location(self, bucket_name): + async def _get_and_set_s3_bucket_location(self, bucket: {}): client = AWSFacadeUtils.get_client('s3', self.session) - location = client.get_bucket_location(Bucket=bucket_name) + location = await run_concurrently(lambda: client.get_bucket_location(Bucket=bucket['Name'])) region = location['LocationConstraint'] if location['LocationConstraint'] else 'us-east-1' # Fixes issue #59: location constraint can be either EU or eu-west-1 for Ireland... if region == 'EU': region = 'eu-west-1' - return region + bucket['region'] = region - async def _set_s3_bucket_logging(self, bucket): + async def _get_and_set_s3_bucket_logging(self, bucket): client = AWSFacadeUtils.get_client('s3', self.session, bucket['region'],) try: logging = await run_concurrently(lambda: client.get_bucket_logging(Bucket=bucket['Name'])) @@ -48,12 +51,12 @@ async def _set_s3_bucket_logging(self, bucket): bucket['logging'] = 'Unknown' if 'LoggingEnabled' in logging: - bucket['logging'] = logging['LoggingEnabled']['TargetBucket'] + '/' + logging['LoggingEnabled']['TargetPrefix'] + bucket['logging'] =\ + logging['LoggingEnabled']['TargetBucket'] + '/' + logging['LoggingEnabled']['TargetPrefix'] else: bucket['logging'] = 'Disabled' - # noinspection PyBroadException - async def _set_s3_bucket_versioning(self, bucket): + async def _get_and_set_s3_bucket_versioning(self, bucket): client = AWSFacadeUtils.get_client('s3', self.session, bucket['region']) try: versioning = await run_concurrently(lambda: client.get_bucket_versioning(Bucket=bucket['Name'])) @@ -63,18 +66,16 @@ async def _set_s3_bucket_versioning(self, bucket): bucket['versioning_status_enabled'] = None bucket['version_mfa_delete_enabled'] = None - - # noinspection PyBroadException - async def _set_s3_bucket_webhosting(self, bucket): + async def _get_and_set_s3_bucket_webhosting(self, bucket): client = AWSFacadeUtils.get_client('s3', self.session, bucket['region']) try: - result = client.get_bucket_website(Bucket=bucket['Name']) + result = await run_concurrently(lambda: client.get_bucket_website(Bucket=bucket['Name'])) bucket['web_hosting_enabled'] = 'IndexDocument' in result except Exception: - # TODO: distinguish permission denied from 'NoSuchWebsiteConfiguration' errors + # TODO: distinguish permission denied from 'NoSuchWebsiteConfiguration' errors bucket['web_hosting_enabled'] = False - async def _set_s3_bucket_default_encryption(self, bucket): + async def _get_and_set_s3_bucket_default_encryption(self, bucket): bucket_name = bucket['Name'] client = AWSFacadeUtils.get_client('s3', self.session, bucket['region']) try: @@ -90,7 +91,7 @@ async def _set_s3_bucket_default_encryption(self, bucket): print_error('Failed to get encryption configuration for %s: %s' % (bucket_name, e)) bucket['default_encryption'] = 'Unknown' - async def _set_s3_acls(self, bucket, key_name=None): + async def _get_and_set_s3_acls(self, bucket, key_name=None): bucket_name = bucket['Name'] client = AWSFacadeUtils.get_client('s3', self.session, bucket['region']) try: @@ -98,7 +99,7 @@ async def _set_s3_acls(self, bucket, key_name=None): if key_name: grants = await run_concurrently(lambda: client.get_object_acl(Bucket=bucket_name, Key=key_name)) else: - grants = await run_concurrently(lambda: client.get_bucket_acl(Bucket=bucket_name)) + grants = await run_concurrently(lambda: client.get_bucket_acl(Bucket=bucket_name)) for grant in grants['Grants']: if 'ID' in grant['Grantee']: grantee = grant['Grantee']['ID'] @@ -121,10 +122,10 @@ async def _set_s3_acls(self, bucket, key_name=None): print_error('Failed to get ACL configuration for %s: %s' % (bucket_name, e)) bucket['grantees'] = {} - async def _set_s3_bucket_policy(self, bucket): + async def _get_and_set_s3_bucket_policy(self, bucket): client = AWSFacadeUtils.get_client('s3', self.session, bucket['region']) try: - bucket_policy = await run_concurrently(lambda: client.get_bucket_policy(Bucket=bucket['Name'])) + bucket_policy = await run_concurrently(lambda: client.get_bucket_policy(Bucket=bucket['Name'])) bucket['policy'] = json.loads(bucket_policy['Policy']) except ClientError as e: if e.response['Error']['Code'] != 'NoSuchBucketPolicy': @@ -182,4 +183,3 @@ def _s3_group_to_string(uri): def _status_to_bool(value): """ Converts a string to True if it is equal to 'Enabled' or to False otherwise. """ return value == 'Enabled' - From e4d9a5ab18bc3abb2a74dd68be1c74049da681ee Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 30 Mar 2019 15:18:37 -0400 Subject: [PATCH 655/667] Improve concurrent support for SES facade. --- ScoutSuite/providers/aws/facade/ses.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/ses.py b/ScoutSuite/providers/aws/facade/ses.py index 8accf84f7..a78edce37 100644 --- a/ScoutSuite/providers/aws/facade/ses.py +++ b/ScoutSuite/providers/aws/facade/ses.py @@ -1,27 +1,17 @@ from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils from ScoutSuite.providers.utils import run_concurrently from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade - -import asyncio +from ScoutSuite.providers.utils import map_concurrently class SESFacade(AWSBaseFacade): async def get_identities(self, region: str): - identity_names = await AWSFacadeUtils.get_all_pages('ses', region, self.session, 'list_identities', 'Identities') - identities = [] - # Fetch dkim attributes concurrently: - tasks = { - asyncio.ensure_future( - self.get_identity_dkim_attributes(region, identity_name) - ) for identity_name in identity_names - } - for result in asyncio.as_completed(tasks): - identity_name, dkim_attributes = await result - identities.append((identity_name, dkim_attributes)) + identity_names = await AWSFacadeUtils.get_all_pages( + 'ses', region, self.session, 'list_identities', 'Identities') - return identities + return await map_concurrently(self._get_identity_dkim_attributes, identity_names, region=region) - async def get_identity_dkim_attributes(self, region: str, identity_name: str): + async def _get_identity_dkim_attributes(self, identity_name: str, region: str): ses_client = AWSFacadeUtils.get_client('ses', self.session, region) dkim_attributes = await run_concurrently( lambda: ses_client.get_identity_dkim_attributes(Identities=[identity_name])['DkimAttributes'][identity_name] From 2b5fbc63c43ef336beb134db07e24025514c6f04 Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 30 Mar 2019 15:18:50 -0400 Subject: [PATCH 656/667] Improve concurrent support for SNS facade. --- ScoutSuite/providers/aws/facade/sns.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/sns.py b/ScoutSuite/providers/aws/facade/sns.py index 24b749aee..ede1542ff 100644 --- a/ScoutSuite/providers/aws/facade/sns.py +++ b/ScoutSuite/providers/aws/facade/sns.py @@ -1,9 +1,9 @@ +import asyncio + from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade from ScoutSuite.providers.utils import run_concurrently -import asyncio - class SNSFacade(AWSBaseFacade): regional_subscriptions_cache_locks = {} @@ -11,21 +11,11 @@ class SNSFacade(AWSBaseFacade): async def get_topics(self, region: str): topics = await AWSFacadeUtils.get_all_pages('sns', region, self.session, 'list_topics', 'Topics') - - if len(topics) == 0: - return [] - - # Fetch and set the attributes of all topics concurrently: - tasks = { - asyncio.ensure_future( - self.get_and_set_topic_attributes(region, topic) - ) for topic in topics - } - await asyncio.wait(tasks) + await AWSFacadeUtils.get_and_set_concurrently([self._get_and_set_topic_attributes], topics, region=region) return topics - async def get_and_set_topic_attributes(self, region: str, topic: {}): + async def _get_and_set_topic_attributes(self, topic: {}, region: str): sns_client = AWSFacadeUtils.get_client('sns', self.session, region) topic['attributes'] = await run_concurrently( lambda: sns_client.get_topic_attributes(TopicArn=topic['TopicArn'])['Attributes'] From d5af00ff3930ae2f8a6bac5ac17da3203969b01c Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 30 Mar 2019 15:19:06 -0400 Subject: [PATCH 657/667] Improve concurrent support for SQS facade. --- ScoutSuite/providers/aws/facade/sqs.py | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/sqs.py b/ScoutSuite/providers/aws/facade/sqs.py index 80ef63a21..a519e1cef 100644 --- a/ScoutSuite/providers/aws/facade/sqs.py +++ b/ScoutSuite/providers/aws/facade/sqs.py @@ -1,8 +1,6 @@ from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils -from ScoutSuite.providers.utils import run_concurrently - -import asyncio +from ScoutSuite.providers.utils import run_concurrently, map_concurrently class SQSFacade(AWSBaseFacade): @@ -12,22 +10,12 @@ async def get_queues(self, region: str, attribute_names: []): if 'QueueUrls' not in raw_queues: return [] - queue_urls = raw_queues['QueueUrls'] - # Fetch the attributes of all the queues concurrently:: - tasks = { - asyncio.ensure_future( - self.get_queue_attributes(region, queue_url, attribute_names) - ) for queue_url in queue_urls - } - queues = [] - for result in asyncio.as_completed(tasks): - queue_url, queue_attributes = await result - queues.append((queue_url, queue_attributes)) - return queues + return await map_concurrently( + self._get_queue_attributes, queue_urls, region=region, attribute_names=attribute_names) - async def get_queue_attributes(self, region: str, queue_url: str, attribute_names: []): + async def _get_queue_attributes(self, queue_url: str, region: str, attribute_names: []): sqs_client = AWSFacadeUtils.get_client('sqs', self.session, region) queue_attributes = await run_concurrently( lambda: sqs_client.get_queue_attributes(QueueUrl=queue_url, AttributeNames=attribute_names)['Attributes'] From 8d890efa6169697b96bf070e3128585bb57dd3f0 Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 30 Mar 2019 15:19:32 -0400 Subject: [PATCH 658/667] Add guard for preprocessing. --- ScoutSuite/providers/aws/provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index 0cfa4b5ee..0db16dbdc 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -72,7 +72,7 @@ def preprocessing(self, ip_ranges=None, ip_ranges_name_key=None): if 'cloudtrail' in self.service_list: self._process_cloudtrail_trails(self.services['cloudtrail']) - if 'elbv2' in self.service_list: + if 'elbv2' in self.service_list and 'ec2' in self.service_list: self._add_security_group_data_to_elbv2() if 's3' in self.service_list and 'iam' in self.service_list: From 731d7b3249d6a19ebd268ba158ad918472a5ddb0 Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 30 Mar 2019 22:54:58 -0400 Subject: [PATCH 659/667] Move get_and_set_concurrently to providers/utils.py --- .../providers/aws/facade/cloudformation.py | 3 +- ScoutSuite/providers/aws/facade/cloudtrail.py | 3 +- ScoutSuite/providers/aws/facade/ec2.py | 5 ++-- ScoutSuite/providers/aws/facade/efs.py | 6 ++-- .../providers/aws/facade/elasticache.py | 3 +- ScoutSuite/providers/aws/facade/elb.py | 4 +-- ScoutSuite/providers/aws/facade/elbv2.py | 4 +-- ScoutSuite/providers/aws/facade/iam.py | 10 +++---- ScoutSuite/providers/aws/facade/rds.py | 8 ++--- ScoutSuite/providers/aws/facade/s3.py | 6 ++-- ScoutSuite/providers/aws/facade/sns.py | 4 +-- ScoutSuite/providers/aws/facade/utils.py | 25 ---------------- ScoutSuite/providers/utils.py | 30 ++++++++++++++++++- 13 files changed, 59 insertions(+), 52 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/cloudformation.py b/ScoutSuite/providers/aws/facade/cloudformation.py index be2f8980c..d4cecd00f 100644 --- a/ScoutSuite/providers/aws/facade/cloudformation.py +++ b/ScoutSuite/providers/aws/facade/cloudformation.py @@ -3,6 +3,7 @@ from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils from ScoutSuite.providers.utils import run_concurrently from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade +from ScoutSuite.providers.utils import get_and_set_concurrently class CloudFormation(AWSBaseFacade): @@ -11,7 +12,7 @@ async def get_stacks(self, region: str): stacks = await AWSFacadeUtils.get_all_pages( 'cloudformation', region, self.session, 'list_stacks', 'StackSummaries') stacks = [stack for stack in stacks if not CloudFormation._is_stack_deleted(stack)] - await AWSFacadeUtils.get_and_set_concurrently( + await get_and_set_concurrently( [self._get_and_set_description, self._get_and_set_template, self._get_and_set_policy], stacks, region=region) diff --git a/ScoutSuite/providers/aws/facade/cloudtrail.py b/ScoutSuite/providers/aws/facade/cloudtrail.py index 6f41cc0cb..fbc0d4c2c 100644 --- a/ScoutSuite/providers/aws/facade/cloudtrail.py +++ b/ScoutSuite/providers/aws/facade/cloudtrail.py @@ -1,6 +1,7 @@ from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils from ScoutSuite.providers.utils import run_concurrently from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade +from ScoutSuite.providers.utils import get_and_set_concurrently class CloudTrailFacade(AWSBaseFacade): @@ -8,7 +9,7 @@ async def get_trails(self, region): client = AWSFacadeUtils.get_client('cloudtrail', self.session, region) trails = await run_concurrently( lambda: client.describe_trails()['trailList']) - await AWSFacadeUtils.get_and_set_concurrently( + await get_and_set_concurrently( [self._get_and_set_status, self._get_and_set_selectors], trails, region=region) return trails diff --git a/ScoutSuite/providers/aws/facade/ec2.py b/ScoutSuite/providers/aws/facade/ec2.py index 852494873..68bf2684b 100644 --- a/ScoutSuite/providers/aws/facade/ec2.py +++ b/ScoutSuite/providers/aws/facade/ec2.py @@ -4,6 +4,7 @@ from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils from ScoutSuite.providers.utils import run_concurrently from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade +from ScoutSuite.providers.utils import get_and_set_concurrently class EC2Facade(AWSBaseFacade): @@ -60,7 +61,7 @@ async def get_snapshots(self, region: str, owner_id: str): filters = [{'Name': 'owner-id', 'Values': [owner_id]}] snapshots = await AWSFacadeUtils.get_all_pages( 'ec2', region, self.session, 'describe_snapshots', 'Snapshots', Filters=filters) - await AWSFacadeUtils.get_and_set_concurrently([self._get_and_set_snapshot_attributes], snapshots, region=region) + await get_and_set_concurrently([self._get_and_set_snapshot_attributes], snapshots, region=region) return snapshots @@ -91,7 +92,7 @@ async def get_subnets(self, region: str, vpc: str): ec2_client = AWSFacadeUtils.get_client('ec2', self.session, region) filters = [{'Name': 'vpc-id', 'Values': [vpc]}] subnets = await run_concurrently(lambda: ec2_client.describe_subnets(Filters=filters)['Subnets']) - await AWSFacadeUtils.get_and_set_concurrently([self._get_and_set_subnet_flow_logs], subnets, region=region) + await get_and_set_concurrently([self._get_and_set_subnet_flow_logs], subnets, region=region) return subnets diff --git a/ScoutSuite/providers/aws/facade/efs.py b/ScoutSuite/providers/aws/facade/efs.py index 8ce9f500f..20b1485a8 100644 --- a/ScoutSuite/providers/aws/facade/efs.py +++ b/ScoutSuite/providers/aws/facade/efs.py @@ -1,5 +1,5 @@ from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils -from ScoutSuite.providers.utils import run_concurrently +from ScoutSuite.providers.utils import run_concurrently, get_and_set_concurrently from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade @@ -7,7 +7,7 @@ class EFSFacade(AWSBaseFacade): async def get_file_systems(self, region: str): file_systems = await AWSFacadeUtils.get_all_pages( 'efs', region, self.session, 'describe_file_systems', 'FileSystems') - await AWSFacadeUtils.get_and_set_concurrently( + await get_and_set_concurrently( [self._get_and_set_tags, self._get_and_set_mount_targets], file_systems, region=region) return file_systems @@ -30,7 +30,7 @@ async def _get_and_set_mount_targets(self, file_system: {}, region: str): mount_target_id = mount_target['MountTargetId'] file_system['MountTargets'][mount_target_id] = mount_target - await AWSFacadeUtils.get_and_set_concurrently( + await get_and_set_concurrently( [self._get_and_set_mount_target_security_groups], mount_targets, region=region) async def _get_and_set_mount_target_security_groups(self, mount_target: {}, region: str): diff --git a/ScoutSuite/providers/aws/facade/elasticache.py b/ScoutSuite/providers/aws/facade/elasticache.py index a1bd9f6cf..b023a72a5 100644 --- a/ScoutSuite/providers/aws/facade/elasticache.py +++ b/ScoutSuite/providers/aws/facade/elasticache.py @@ -1,6 +1,7 @@ from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade from ScoutSuite.providers.aws.utils import ec2_classic +from ScoutSuite.providers.utils import get_and_set_concurrently from asyncio import Lock @@ -22,7 +23,7 @@ async def cache_clusters(self, region): self.clusters_cache[region] = await AWSFacadeUtils.get_all_pages( 'elasticache', region, self.session, 'describe_cache_clusters', 'CacheClusters') - await AWSFacadeUtils.get_and_set_concurrently( + await get_and_set_concurrently( [self._get_and_set_cluster_vpc], self.clusters_cache[region], region=region) async def _get_and_set_cluster_vpc(self, cluster: {}, region: str): diff --git a/ScoutSuite/providers/aws/facade/elb.py b/ScoutSuite/providers/aws/facade/elb.py index 179c75651..c71e72120 100644 --- a/ScoutSuite/providers/aws/facade/elb.py +++ b/ScoutSuite/providers/aws/facade/elb.py @@ -1,7 +1,7 @@ from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade from ScoutSuite.providers.aws.utils import ec2_classic -from ScoutSuite.providers.utils import run_concurrently +from ScoutSuite.providers.utils import run_concurrently, get_and_set_concurrently import asyncio @@ -27,7 +27,7 @@ async def cache_load_balancers(self, region): load_balancer['VpcId'] =\ load_balancer['VPCId'] if 'VPCId' in load_balancer and load_balancer['VPCId'] else ec2_classic - await AWSFacadeUtils.get_and_set_concurrently( + await get_and_set_concurrently( [self._get_and_set_load_balancer_attributes], self.load_balancers_cache[region], region=region) async def _get_and_set_load_balancer_attributes(self, load_balancer: {}, region: str): diff --git a/ScoutSuite/providers/aws/facade/elbv2.py b/ScoutSuite/providers/aws/facade/elbv2.py index 8368dab16..f75b62169 100644 --- a/ScoutSuite/providers/aws/facade/elbv2.py +++ b/ScoutSuite/providers/aws/facade/elbv2.py @@ -3,7 +3,7 @@ from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade from ScoutSuite.providers.aws.utils import ec2_classic -from ScoutSuite.providers.utils import run_concurrently +from ScoutSuite.providers.utils import run_concurrently, get_and_set_concurrently class ELBv2Facade(AWSBaseFacade): @@ -27,7 +27,7 @@ async def cache_load_balancers(self, region): load_balancer['VpcId'] =\ load_balancer['VpcId'] if 'VpcId' in load_balancer and load_balancer['VpcId'] else ec2_classic - await AWSFacadeUtils.get_and_set_concurrently( + await get_and_set_concurrently( [self._get_and_set_load_balancer_attributes], self.load_balancers_cache[region], region=region) async def _get_and_set_load_balancer_attributes(self, load_balancer: dict, region: str): diff --git a/ScoutSuite/providers/aws/facade/iam.py b/ScoutSuite/providers/aws/facade/iam.py index 29db39a98..e6c7ccfb6 100644 --- a/ScoutSuite/providers/aws/facade/iam.py +++ b/ScoutSuite/providers/aws/facade/iam.py @@ -4,7 +4,7 @@ from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade from ScoutSuite.core.console import print_error -from ScoutSuite.providers.utils import get_non_provider_id, run_concurrently +from ScoutSuite.providers.utils import get_non_provider_id, run_concurrently, get_and_set_concurrently class IAMFacade(AWSBaseFacade): @@ -36,7 +36,7 @@ async def get_credential_reports(self): async def get_groups(self): groups = await AWSFacadeUtils.get_all_pages('iam', None, self.session, 'list_groups', 'Groups') - await AWSFacadeUtils.get_and_set_concurrently( + await get_and_set_concurrently( [self._get_and_set_group_users, functools.partial(self._get_and_set_inline_policies, iam_resource_type='group')], groups) return groups @@ -44,7 +44,7 @@ async def get_groups(self): async def get_policies(self): policies = await AWSFacadeUtils.get_all_pages( 'iam', None, self.session, 'list_policies', 'Policies', OnlyAttached=True) - await AWSFacadeUtils.get_and_set_concurrently([self._get_and_set_policy_details], policies) + await get_and_set_concurrently([self._get_and_set_policy_details], policies) return policies async def _get_and_set_policy_details(self, policy): @@ -74,7 +74,7 @@ async def _get_and_set_policy_details(self, policy): async def get_users(self): users = await AWSFacadeUtils.get_all_pages('iam', None, self.session, 'list_users', 'Users') - await AWSFacadeUtils.get_and_set_concurrently( + await get_and_set_concurrently( [functools.partial(self._get_and_set_inline_policies, iam_resource_type='user'), self._get_and_set_user_groups, self._get_and_set_user_login_profile, @@ -104,7 +104,7 @@ async def get_roles(self): role['assume_role_policy'] = {} role['assume_role_policy']['PolicyDocument'] = role.pop( 'AssumeRolePolicyDocument') - await AWSFacadeUtils.get_and_set_concurrently( + await get_and_set_concurrently( [functools.partial(self._get_and_set_inline_policies, iam_resource_type='role'), self._get_and_set_role_profiles], roles) diff --git a/ScoutSuite/providers/aws/facade/rds.py b/ScoutSuite/providers/aws/facade/rds.py index 9fdaf9931..72e947dd4 100644 --- a/ScoutSuite/providers/aws/facade/rds.py +++ b/ScoutSuite/providers/aws/facade/rds.py @@ -3,7 +3,7 @@ from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade from ScoutSuite.providers.aws.utils import ec2_classic -from ScoutSuite.providers.utils import run_concurrently +from ScoutSuite.providers.utils import run_concurrently, get_and_set_concurrently from ScoutSuite.core.console import print_error, print_exception @@ -33,7 +33,7 @@ async def _cache_instances(self, region: str): and instance['DBSubnetGroup']['VpcId'] \ else ec2_classic - await AWSFacadeUtils.get_and_set_concurrently( + await get_and_set_concurrently( [self._get_and_set_instance_clusters], self._instances_cache[region], region=region) async def _get_and_set_instance_clusters(self, instance: {}, region: str): @@ -60,7 +60,7 @@ async def _cache_snapshots(self, region: str): for snapshot in self._snapshots_cache[region]: snapshot['VpcId'] = snapshot['VpcId'] if 'VpcId' in snapshot else ec2_classic - await AWSFacadeUtils.get_and_set_concurrently( + await get_and_set_concurrently( [self._get_and_set_snapshot_attributes], self._snapshots_cache[region], region=region) async def _get_and_set_snapshot_attributes(self, snapshot: {}, region: str): @@ -86,7 +86,7 @@ async def _cache_subnet_groups(self, region: str): async def get_parameter_groups(self, region: str): parameter_groups = await AWSFacadeUtils.get_all_pages( 'rds', region, self.session, 'describe_db_parameter_groups', 'DBParameterGroups') - await AWSFacadeUtils.get_and_set_concurrently( + await get_and_set_concurrently( [self._get_and_set_db_parameters], parameter_groups, region=region) return parameter_groups diff --git a/ScoutSuite/providers/aws/facade/s3.py b/ScoutSuite/providers/aws/facade/s3.py index 5e6b99a4c..cb222f5c2 100644 --- a/ScoutSuite/providers/aws/facade/s3.py +++ b/ScoutSuite/providers/aws/facade/s3.py @@ -3,7 +3,7 @@ from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade -from ScoutSuite.providers.utils import run_concurrently +from ScoutSuite.providers.utils import run_concurrently, get_and_set_concurrently from ScoutSuite.core.console import print_error @@ -13,10 +13,10 @@ async def get_buckets(self): buckets = await run_concurrently(lambda: client.list_buckets()['Buckets']) # We need first to retrieve bucket locations before retrieving bucket details: - await AWSFacadeUtils.get_and_set_concurrently([self._get_and_set_s3_bucket_location], buckets) + await get_and_set_concurrently([self._get_and_set_s3_bucket_location], buckets) # Then we can retrieve bucket details concurrently: - await AWSFacadeUtils.get_and_set_concurrently( + await get_and_set_concurrently( [self._get_and_set_s3_bucket_logging, self._get_and_set_s3_bucket_versioning, self._get_and_set_s3_bucket_webhosting, diff --git a/ScoutSuite/providers/aws/facade/sns.py b/ScoutSuite/providers/aws/facade/sns.py index ede1542ff..98d1b4789 100644 --- a/ScoutSuite/providers/aws/facade/sns.py +++ b/ScoutSuite/providers/aws/facade/sns.py @@ -2,7 +2,7 @@ from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade -from ScoutSuite.providers.utils import run_concurrently +from ScoutSuite.providers.utils import run_concurrently, get_and_set_concurrently class SNSFacade(AWSBaseFacade): @@ -11,7 +11,7 @@ class SNSFacade(AWSBaseFacade): async def get_topics(self, region: str): topics = await AWSFacadeUtils.get_all_pages('sns', region, self.session, 'list_topics', 'Topics') - await AWSFacadeUtils.get_and_set_concurrently([self._get_and_set_topic_attributes], topics, region=region) + await get_and_set_concurrently([self._get_and_set_topic_attributes], topics, region=region) return topics diff --git a/ScoutSuite/providers/aws/facade/utils.py b/ScoutSuite/providers/aws/facade/utils.py index 86193adb8..eb81a24d1 100644 --- a/ScoutSuite/providers/aws/facade/utils.py +++ b/ScoutSuite/providers/aws/facade/utils.py @@ -78,28 +78,3 @@ def get_client(service: str, session: boto3.session.Session, region: str = None) return AWSFacadeUtils._clients.setdefault( (service, region), session.client(service, region_name=region) if region else session.client(service)) - - @staticmethod - async def get_and_set_concurrently(get_and_set_funcs: [], entities: [], **kwargs): - """ - Given a list of get_and_set_* functions (ex: get_and_set_description, get_and_set_attributes, - get_and_set_policy, etc.) and a list of entities (ex: stacks, keys, load balancers, vpcs, etc.), - get_and_set_concurrently will call each of these functions concurrently on each entity. - - :param get_and_set_funcs: list of functions that takes a region and an entity (they must have the following - signature: region: str, entity: {}) and then fetch and set some kind of attributes to this entity. - :param entities: list of a same kind of entities - :param region: a region - - :return: - """ - - if len(entities) == 0: - return - - tasks = { - asyncio.ensure_future( - get_and_set_func(entity, **kwargs) - ) for entity in entities for get_and_set_func in get_and_set_funcs - } - await asyncio.wait(tasks) diff --git a/ScoutSuite/providers/utils.py b/ScoutSuite/providers/utils.py index bcedcc3d7..1eb942832 100644 --- a/ScoutSuite/providers/utils.py +++ b/ScoutSuite/providers/utils.py @@ -28,6 +28,31 @@ def run_concurrently(func): return asyncio.get_event_loop().run_in_executor(executor=None, func=func) +async def get_and_set_concurrently(get_and_set_funcs: [], entities: [], **kwargs): + """ + Given a list of get_and_set_* functions (ex: get_and_set_description, get_and_set_attributes, + get_and_set_policy, etc.) and a list of entities (ex: stacks, keys, load balancers, vpcs, etc.), + get_and_set_concurrently will call each of these functions concurrently on each entity. + + :param get_and_set_funcs: list of functions that takes a region and an entity (they must have the following + signature: region: str, entity: {}) and then fetch and set some kind of attributes to this entity. + :param entities: list of a same kind of entities + :param region: a region + + :return: + """ + + if len(entities) == 0: + return + + tasks = { + asyncio.ensure_future( + get_and_set_func(entity, **kwargs) + ) for entity in entities for get_and_set_func in get_and_set_funcs + } + await asyncio.wait(tasks) + + async def map_concurrently(coro, entities, **kwargs): """ Given a list of entities, executes coroutine `coro` concurrently on each entity and returns a list of the obtained @@ -38,8 +63,11 @@ async def map_concurrently(coro, entities, **kwargs): :return: a list of new entities (ex: clusters) """ - results = [] + if len(entities) == 0: + return [] + + results = [] tasks = { asyncio.ensure_future( coro(entity, **kwargs) From 2d72e795791d0f93aca33585f13a6e67db0747e7 Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 30 Mar 2019 23:09:06 -0400 Subject: [PATCH 660/667] Fix indentation, remove useless imports, fix PEP8 issues. --- ScoutSuite/providers/aws/facade/basefacade.py | 1 - ScoutSuite/providers/aws/facade/elb.py | 4 ++-- ScoutSuite/providers/aws/facade/facade.py | 1 - ScoutSuite/providers/aws/facade/redshift.py | 4 ++-- ScoutSuite/providers/aws/facade/utils.py | 1 - .../aws/resources/cloudtrail/service.py | 6 +++-- .../providers/aws/resources/ec2/instances.py | 3 ++- .../providers/aws/resources/ec2/service.py | 19 +++++++++------ .../providers/aws/resources/ec2/vpcs.py | 2 -- .../aws/resources/elasticache/cluster.py | 7 +++--- .../resources/elasticache/parametergroups.py | 2 +- .../aws/resources/elasticache/subnetgroups.py | 7 +++--- .../aws/resources/elasticache/vpcs.py | 5 +--- .../aws/resources/iam/credentialreports.py | 8 ++++--- .../providers/aws/resources/iam/policies.py | 1 + .../providers/aws/resources/iam/service.py | 24 +++++++++---------- .../aws/resources/rds/parametergroups.py | 2 +- .../providers/aws/resources/rds/service.py | 9 ++++--- .../providers/aws/resources/rds/snapshots.py | 1 - .../aws/resources/rds/subnetgroups.py | 1 - ScoutSuite/providers/aws/resources/regions.py | 1 - .../providers/aws/resources/s3/service.py | 1 - .../aws/resources/ses/identity_policies.py | 4 ++-- .../providers/aws/resources/sns/topics.py | 4 ++-- 24 files changed, 59 insertions(+), 59 deletions(-) diff --git a/ScoutSuite/providers/aws/facade/basefacade.py b/ScoutSuite/providers/aws/facade/basefacade.py index 0725be995..47de75ae3 100644 --- a/ScoutSuite/providers/aws/facade/basefacade.py +++ b/ScoutSuite/providers/aws/facade/basefacade.py @@ -4,4 +4,3 @@ class AWSBaseFacade(object): def __init__(self, session: boto3.session.Session): self.session = session - \ No newline at end of file diff --git a/ScoutSuite/providers/aws/facade/elb.py b/ScoutSuite/providers/aws/facade/elb.py index c71e72120..28c73b504 100644 --- a/ScoutSuite/providers/aws/facade/elb.py +++ b/ScoutSuite/providers/aws/facade/elb.py @@ -1,10 +1,10 @@ +import asyncio + from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade from ScoutSuite.providers.aws.utils import ec2_classic from ScoutSuite.providers.utils import run_concurrently, get_and_set_concurrently -import asyncio - class ELBFacade(AWSBaseFacade): regional_load_balancers_cache_locks = {} diff --git a/ScoutSuite/providers/aws/facade/facade.py b/ScoutSuite/providers/aws/facade/facade.py index 2e84b0a95..708bef6a0 100644 --- a/ScoutSuite/providers/aws/facade/facade.py +++ b/ScoutSuite/providers/aws/facade/facade.py @@ -2,7 +2,6 @@ from collections import Counter from botocore.session import Session -from ScoutSuite.providers.utils import run_concurrently from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade from ScoutSuite.providers.aws.facade.awslambda import LambdaFacade from ScoutSuite.providers.aws.facade.cloudformation import CloudFormation diff --git a/ScoutSuite/providers/aws/facade/redshift.py b/ScoutSuite/providers/aws/facade/redshift.py index 3ffa0c84b..525b438b5 100644 --- a/ScoutSuite/providers/aws/facade/redshift.py +++ b/ScoutSuite/providers/aws/facade/redshift.py @@ -1,10 +1,10 @@ +from asyncio import Lock + from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade from ScoutSuite.providers.aws.utils import ec2_classic from botocore.utils import ClientError -from asyncio import Lock - class RedshiftFacade(AWSBaseFacade): regional_cluster_cache_locks = {} diff --git a/ScoutSuite/providers/aws/facade/utils.py b/ScoutSuite/providers/aws/facade/utils.py index eb81a24d1..0cced40bf 100644 --- a/ScoutSuite/providers/aws/facade/utils.py +++ b/ScoutSuite/providers/aws/facade/utils.py @@ -1,5 +1,4 @@ import boto3 -import asyncio from ScoutSuite.providers.utils import run_concurrently diff --git a/ScoutSuite/providers/aws/resources/cloudtrail/service.py b/ScoutSuite/providers/aws/resources/cloudtrail/service.py index 1e2c554d3..d0109eb66 100644 --- a/ScoutSuite/providers/aws/resources/cloudtrail/service.py +++ b/ScoutSuite/providers/aws/resources/cloudtrail/service.py @@ -18,7 +18,8 @@ def _parse_trail(self, raw_trail): trail_id = get_non_provider_id(trail['name']) # Do not duplicate entries for multiregion trails - if 'IsMultiRegionTrail' in raw_trail and raw_trail['IsMultiRegionTrail'] and raw_trail['HomeRegion'] != self.scope['region']: + if 'IsMultiRegionTrail' in raw_trail and raw_trail['IsMultiRegionTrail'] and \ + raw_trail['HomeRegion'] != self.scope['region']: for key in ['HomeRegion', 'TrailARN']: trail[key] = raw_trail[key] trail['scout2_link'] = 'services.cloudtrail.regions.%s.trails.%s' % (raw_trail['HomeRegion'], trail_id) @@ -47,7 +48,8 @@ def _parse_trail(self, raw_trail): def data_logging_status(self, trail): for event_selector in trail['EventSelectors']: - has_wildcard = {u'Values': [u'arn:aws:s3:::'], u'Type': u'AWS::S3::Object'} in event_selector['DataResources'] + has_wildcard =\ + {u'Values': [u'arn:aws:s3:::'], u'Type': u'AWS::S3::Object'} in event_selector['DataResources'] is_logging = trail['IsLogging'] if has_wildcard and is_logging and self.is_fresh(trail): diff --git a/ScoutSuite/providers/aws/resources/ec2/instances.py b/ScoutSuite/providers/aws/resources/ec2/instances.py index ef999f427..45c630138 100644 --- a/ScoutSuite/providers/aws/resources/ec2/instances.py +++ b/ScoutSuite/providers/aws/resources/ec2/instances.py @@ -21,7 +21,8 @@ async def _parse_instance(self, raw_instance): instance['user_data_secrets'] = self._identify_user_data_secrets(instance['user_data']) get_name(raw_instance, instance, 'InstanceId') - get_keys(raw_instance, instance, ['KeyName', 'LaunchTime', 'InstanceType', 'State', 'IamInstanceProfile', 'SubnetId']) + get_keys(raw_instance, instance, + ['KeyName', 'LaunchTime', 'InstanceType', 'State', 'IamInstanceProfile', 'SubnetId']) instance['network_interfaces'] = {} for eni in raw_instance['NetworkInterfaces']: diff --git a/ScoutSuite/providers/aws/resources/ec2/service.py b/ScoutSuite/providers/aws/resources/ec2/service.py index 35b798d17..ab3ef2572 100644 --- a/ScoutSuite/providers/aws/resources/ec2/service.py +++ b/ScoutSuite/providers/aws/resources/ec2/service.py @@ -1,4 +1,3 @@ -from ScoutSuite.providers.aws.facade.facade import AWSFacade from ScoutSuite.providers.aws.resources.ec2.ami import AmazonMachineImages from ScoutSuite.providers.aws.resources.ec2.snapshots import Snapshots from ScoutSuite.providers.aws.resources.ec2.volumes import Volumes @@ -21,10 +20,16 @@ async def fetch_all(self, credentials=None, regions=None, partition_name='aws'): await super(EC2, self).fetch_all(credentials, regions, partition_name) for region in self['regions']: - self['regions'][region]['instances_count'] = sum([len(vpc['instances']) for vpc in self['regions'][region]['vpcs'].values()]) - self['regions'][region]['security_groups_count'] = sum([len(vpc['security_groups']) for vpc in self['regions'][region]['vpcs'].values()]) - self['regions'][region]['network_interfaces_count'] = sum([len(vpc['network_interfaces']) for vpc in self['regions'][region]['vpcs'].values()]) + self['regions'][region]['instances_count'] =\ + sum([len(vpc['instances']) for vpc in self['regions'][region]['vpcs'].values()]) + self['regions'][region]['security_groups_count'] =\ + sum([len(vpc['security_groups']) for vpc in self['regions'][region]['vpcs'].values()]) + self['regions'][region]['network_interfaces_count'] =\ + sum([len(vpc['network_interfaces']) for vpc in self['regions'][region]['vpcs'].values()]) - self['instances_count'] = sum([region['instances_count'] for region in self['regions'].values()]) - self['security_groups_count'] = sum([region['security_groups_count'] for region in self['regions'].values()]) - self['network_interfaces_count'] = sum([region['network_interfaces_count'] for region in self['regions'].values()]) + self['instances_count'] =\ + sum([region['instances_count'] for region in self['regions'].values()]) + self['security_groups_count'] =\ + sum([region['security_groups_count'] for region in self['regions'].values()]) + self['network_interfaces_count'] =\ + sum([region['network_interfaces_count'] for region in self['regions'].values()]) diff --git a/ScoutSuite/providers/aws/resources/ec2/vpcs.py b/ScoutSuite/providers/aws/resources/ec2/vpcs.py index 28ec102e4..2ece04de3 100644 --- a/ScoutSuite/providers/aws/resources/ec2/vpcs.py +++ b/ScoutSuite/providers/aws/resources/ec2/vpcs.py @@ -1,5 +1,3 @@ -import asyncio - from ScoutSuite.providers.aws.resources.vpcs import Vpcs from ScoutSuite.providers.aws.resources.ec2.instances import EC2Instances from ScoutSuite.providers.aws.resources.ec2.securitygroups import SecurityGroups diff --git a/ScoutSuite/providers/aws/resources/elasticache/cluster.py b/ScoutSuite/providers/aws/resources/elasticache/cluster.py index 2ac367378..2b6f1af04 100644 --- a/ScoutSuite/providers/aws/resources/elasticache/cluster.py +++ b/ScoutSuite/providers/aws/resources/elasticache/cluster.py @@ -1,8 +1,7 @@ -from ScoutSuite.providers.aws.resources.resources import AWSCompositeResources -from ScoutSuite.providers.aws.utils import ec2_classic +from ScoutSuite.providers.aws.resources.resources import AWSResources -class Clusters(AWSCompositeResources): +class Clusters(AWSResources): async def fetch_all(self, **kwargs): raw_clusters = await self.facade.elasticache.get_clusters(self.scope['region'], self.scope['vpc']) for raw_cluster in raw_clusters: @@ -11,4 +10,4 @@ async def fetch_all(self, **kwargs): def _parse_cluster(self, raw_cluster): raw_cluster['name'] = raw_cluster.pop('CacheClusterId') - return raw_cluster['name'], raw_cluster \ No newline at end of file + return raw_cluster['name'], raw_cluster diff --git a/ScoutSuite/providers/aws/resources/elasticache/parametergroups.py b/ScoutSuite/providers/aws/resources/elasticache/parametergroups.py index fb7223d17..57b00b584 100644 --- a/ScoutSuite/providers/aws/resources/elasticache/parametergroups.py +++ b/ScoutSuite/providers/aws/resources/elasticache/parametergroups.py @@ -10,4 +10,4 @@ async def fetch_all(self, **kwargs): def _parse_parameter_group(self, raw_parameter_group): raw_parameter_group['name'] = raw_parameter_group.pop('CacheParameterGroupName') - return raw_parameter_group['name'], raw_parameter_group \ No newline at end of file + return raw_parameter_group['name'], raw_parameter_group diff --git a/ScoutSuite/providers/aws/resources/elasticache/subnetgroups.py b/ScoutSuite/providers/aws/resources/elasticache/subnetgroups.py index 8ef864cb7..64cf03ddd 100644 --- a/ScoutSuite/providers/aws/resources/elasticache/subnetgroups.py +++ b/ScoutSuite/providers/aws/resources/elasticache/subnetgroups.py @@ -1,8 +1,7 @@ -from ScoutSuite.providers.aws.resources.resources import AWSCompositeResources -from ScoutSuite.providers.aws.utils import ec2_classic +from ScoutSuite.providers.aws.resources.resources import AWSResources -class SubnetGroups(AWSCompositeResources): +class SubnetGroups(AWSResources): async def fetch_all(self, **kwargs): raw_subnet_groups = await self.facade.elasticache.get_subnet_groups(self.scope['region'], self.scope['vpc']) for raw_subnet_group in raw_subnet_groups: @@ -11,4 +10,4 @@ async def fetch_all(self, **kwargs): def _parse_subnet_group(self, raw_subnet_group): raw_subnet_group['name'] = raw_subnet_group.pop('CacheSubnetGroupName') - return raw_subnet_group['name'], raw_subnet_group \ No newline at end of file + return raw_subnet_group['name'], raw_subnet_group diff --git a/ScoutSuite/providers/aws/resources/elasticache/vpcs.py b/ScoutSuite/providers/aws/resources/elasticache/vpcs.py index 666c6bb7b..39c26c624 100644 --- a/ScoutSuite/providers/aws/resources/elasticache/vpcs.py +++ b/ScoutSuite/providers/aws/resources/elasticache/vpcs.py @@ -1,9 +1,6 @@ -import asyncio - from ScoutSuite.providers.aws.resources.vpcs import Vpcs from ScoutSuite.providers.aws.resources.elasticache.cluster import Clusters from ScoutSuite.providers.aws.resources.elasticache.subnetgroups import SubnetGroups -from ScoutSuite.providers.aws.utils import ec2_classic class ElastiCacheVpcs(Vpcs): @@ -13,4 +10,4 @@ class ElastiCacheVpcs(Vpcs): ] def __init__(self, facade, scope: dict): - super(ElastiCacheVpcs, self).__init__(facade, scope, add_ec2_classic=True) \ No newline at end of file + super(ElastiCacheVpcs, self).__init__(facade, scope, add_ec2_classic=True) diff --git a/ScoutSuite/providers/aws/resources/iam/credentialreports.py b/ScoutSuite/providers/aws/resources/iam/credentialreports.py index 683a6d5c9..be7bd057c 100644 --- a/ScoutSuite/providers/aws/resources/iam/credentialreports.py +++ b/ScoutSuite/providers/aws/resources/iam/credentialreports.py @@ -13,8 +13,10 @@ def _parse_credential_reports(self, raw_credential_report): raw_credential_report['name'] = user_id raw_credential_report['id'] = user_id raw_credential_report['password_last_used'] = self._sanitize_date(raw_credential_report['password_last_used']) - raw_credential_report['access_key_1_last_used_date'] = self._sanitize_date(raw_credential_report['access_key_1_last_used_date']) - raw_credential_report['access_key_2_last_used_date'] = self._sanitize_date(raw_credential_report['access_key_2_last_used_date']) + raw_credential_report['access_key_1_last_used_date'] =\ + self._sanitize_date(raw_credential_report['access_key_1_last_used_date']) + raw_credential_report['access_key_2_last_used_date'] =\ + self._sanitize_date(raw_credential_report['access_key_2_last_used_date']) raw_credential_report['last_used'] = self._compute_last_used(raw_credential_report) return user_id, raw_credential_report @@ -32,4 +34,4 @@ def _compute_last_used(credential_report): credential_report['access_key_2_last_used_date']] dates = [date for date in dates if date is not None] - return max(dates) if len(dates) > 0 else None \ No newline at end of file + return max(dates) if len(dates) > 0 else None diff --git a/ScoutSuite/providers/aws/resources/iam/policies.py b/ScoutSuite/providers/aws/resources/iam/policies.py index b1a6c6f42..4afa51fe7 100644 --- a/ScoutSuite/providers/aws/resources/iam/policies.py +++ b/ScoutSuite/providers/aws/resources/iam/policies.py @@ -1,5 +1,6 @@ from ScoutSuite.providers.aws.resources.resources import AWSResources + class Policies(AWSResources): async def fetch_all(self, **kwargs): raw_policies = await self.facade.iam.get_policies() diff --git a/ScoutSuite/providers/aws/resources/iam/service.py b/ScoutSuite/providers/aws/resources/iam/service.py index b68de7770..e26caa2bc 100644 --- a/ScoutSuite/providers/aws/resources/iam/service.py +++ b/ScoutSuite/providers/aws/resources/iam/service.py @@ -1,5 +1,3 @@ -import asyncio - from ScoutSuite.providers.aws.resources.resources import AWSCompositeResources from ScoutSuite.providers.aws.resources.iam.credentialreports import CredentialReports from ScoutSuite.providers.aws.resources.iam.groups import Groups @@ -50,7 +48,8 @@ async def finalize(self): entities[entity['id']].setdefault('policies_counts', 0) entities[entity['id']]['policies'].append(policy_id) entities[entity['id']]['policies_counts'] += 1 - self._parse_permissions(policy_id, policy['PolicyDocument'], 'policies', entity_type, entity['id']) + self._parse_permissions( + policy_id, policy['PolicyDocument'], 'policies', entity_type, entity['id']) else: self._parse_permissions( policy_id, policy['PolicyDocument'], 'policies', None, None) @@ -118,14 +117,15 @@ def _parse_action(self, effect, action_string, action, resource_string, resource self._parse_resource(effect, action_string, action, resource_string, resource, iam_resource_type, iam_resource_name, policy_name, policy_type, condition) - def _parse_resource(self, effect, action_string, action, resource_string, resource, iam_resource_type, iam_resource_name, policy_name, policy_type, condition): + def _parse_resource(self, effect, action_string, action, resource_string, resource, iam_resource_type, + iam_resource_name, policy_name, policy_type, condition): self['permissions'][action_string][action][iam_resource_type][effect][iam_resource_name].setdefault( resource_string, {}) - self['permissions'][action_string][action][iam_resource_type][effect][iam_resource_name][resource_string].setdefault(resource, { - }) - self['permissions'][action_string][action][iam_resource_type][effect][iam_resource_name][resource_string][resource].setdefault( - policy_type, {}) - self['permissions'][action_string][action][iam_resource_type][effect][ - iam_resource_name][resource_string][resource][policy_type].setdefault(policy_name, {}) - self['permissions'][action_string][action][iam_resource_type][effect][iam_resource_name][ - resource_string][resource][policy_type][policy_name].setdefault('condition', condition) + self['permissions'][action_string][action][iam_resource_type][effect][iam_resource_name][resource_string].\ + setdefault(resource, {}) + self['permissions'][action_string][action][iam_resource_type][effect][iam_resource_name][resource_string][ + resource].setdefault(policy_type, {}) + self['permissions'][action_string][action][iam_resource_type][effect][iam_resource_name][resource_string][ + resource][policy_type].setdefault(policy_name, {}) + self['permissions'][action_string][action][iam_resource_type][effect][iam_resource_name][resource_string][ + resource][policy_type][policy_name].setdefault('condition', condition) diff --git a/ScoutSuite/providers/aws/resources/rds/parametergroups.py b/ScoutSuite/providers/aws/resources/rds/parametergroups.py index be37b63c2..5efdc4d10 100644 --- a/ScoutSuite/providers/aws/resources/rds/parametergroups.py +++ b/ScoutSuite/providers/aws/resources/rds/parametergroups.py @@ -14,4 +14,4 @@ def _parse_parameter_group(self, raw_parameter_group): raw_parameter_group['name'] = raw_parameter_group.pop('DBParameterGroupName') raw_parameter_group['parameters'] = raw_parameter_group.pop('Parameters') parameter_group_id = get_non_provider_id(raw_parameter_group['name']) - return parameter_group_id, raw_parameter_group \ No newline at end of file + return parameter_group_id, raw_parameter_group diff --git a/ScoutSuite/providers/aws/resources/rds/service.py b/ScoutSuite/providers/aws/resources/rds/service.py index bc8827a1d..29dd3ae6a 100644 --- a/ScoutSuite/providers/aws/resources/rds/service.py +++ b/ScoutSuite/providers/aws/resources/rds/service.py @@ -19,9 +19,12 @@ async def fetch_all(self, credentials=None, regions=None, partition_name='aws'): await super(RDS, self).fetch_all(credentials, regions, partition_name) for region in self['regions']: - self['regions'][region]['instances_count'] = sum([len(vpc['instances']) for vpc in self['regions'][region]['vpcs'].values()]) - self['regions'][region]['snapshots_count'] = sum([len(vpc['snapshots']) for vpc in self['regions'][region]['vpcs'].values()]) - self['regions'][region]['subnet_groups_count'] = sum([len(vpc['subnet_groups']) for vpc in self['regions'][region]['vpcs'].values()]) + self['regions'][region]['instances_count'] =\ + sum([len(vpc['instances']) for vpc in self['regions'][region]['vpcs'].values()]) + self['regions'][region]['snapshots_count'] =\ + sum([len(vpc['snapshots']) for vpc in self['regions'][region]['vpcs'].values()]) + self['regions'][region]['subnet_groups_count'] =\ + sum([len(vpc['subnet_groups']) for vpc in self['regions'][region]['vpcs'].values()]) self['instances_count'] = sum([region['instances_count'] for region in self['regions'].values()]) self['snapshots_count'] = sum([region['snapshots_count'] for region in self['regions'].values()]) diff --git a/ScoutSuite/providers/aws/resources/rds/snapshots.py b/ScoutSuite/providers/aws/resources/rds/snapshots.py index f7988b259..afeede437 100644 --- a/ScoutSuite/providers/aws/resources/rds/snapshots.py +++ b/ScoutSuite/providers/aws/resources/rds/snapshots.py @@ -1,5 +1,4 @@ from ScoutSuite.providers.aws.resources.resources import AWSResources -from ScoutSuite.providers.utils import get_non_provider_id class Snapshots(AWSResources): diff --git a/ScoutSuite/providers/aws/resources/rds/subnetgroups.py b/ScoutSuite/providers/aws/resources/rds/subnetgroups.py index 68999fd7d..c3a0d8cd0 100644 --- a/ScoutSuite/providers/aws/resources/rds/subnetgroups.py +++ b/ScoutSuite/providers/aws/resources/rds/subnetgroups.py @@ -1,4 +1,3 @@ -from ScoutSuite.providers.aws.resources.regions import Regions from ScoutSuite.providers.aws.resources.resources import AWSResources diff --git a/ScoutSuite/providers/aws/resources/regions.py b/ScoutSuite/providers/aws/resources/regions.py index 28451fbf5..857095437 100644 --- a/ScoutSuite/providers/aws/resources/regions.py +++ b/ScoutSuite/providers/aws/resources/regions.py @@ -1,5 +1,4 @@ import abc -import asyncio from ScoutSuite.providers.aws.utils import get_aws_account_id from ScoutSuite.providers.aws.resources.resources import AWSCompositeResources diff --git a/ScoutSuite/providers/aws/resources/s3/service.py b/ScoutSuite/providers/aws/resources/s3/service.py index bcacd7064..0d7e94c07 100644 --- a/ScoutSuite/providers/aws/resources/s3/service.py +++ b/ScoutSuite/providers/aws/resources/s3/service.py @@ -1,5 +1,4 @@ from ScoutSuite.providers.aws.facade.facade import AWSFacade -from ScoutSuite.providers.aws.resources.regions import Regions from ScoutSuite.providers.aws.resources.resources import AWSResources, AWSCompositeResources from ScoutSuite.providers.utils import get_non_provider_id diff --git a/ScoutSuite/providers/aws/resources/ses/identity_policies.py b/ScoutSuite/providers/aws/resources/ses/identity_policies.py index 0548506f1..9e720623c 100644 --- a/ScoutSuite/providers/aws/resources/ses/identity_policies.py +++ b/ScoutSuite/providers/aws/resources/ses/identity_policies.py @@ -1,7 +1,7 @@ -from ScoutSuite.providers.aws.resources.resources import AWSResources - import json +from ScoutSuite.providers.aws.resources.resources import AWSResources + class IdentityPolicies(AWSResources): async def fetch_all(self, **kwargs): diff --git a/ScoutSuite/providers/aws/resources/sns/topics.py b/ScoutSuite/providers/aws/resources/sns/topics.py index 57cc51d2f..0f3b2f720 100644 --- a/ScoutSuite/providers/aws/resources/sns/topics.py +++ b/ScoutSuite/providers/aws/resources/sns/topics.py @@ -1,9 +1,9 @@ +import json + from ScoutSuite.providers.aws.resources.resources import AWSCompositeResources from .subscriptions import Subscriptions -import json - class Topics(AWSCompositeResources): _children = [ From c569d884f4d0f802c81986d2cce672815277c977 Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 30 Mar 2019 23:24:54 -0400 Subject: [PATCH 661/667] Fix PEP8 issue. --- ScoutSuite/providers/azure/facade/facade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/azure/facade/facade.py b/ScoutSuite/providers/azure/facade/facade.py index 9344db261..e1dd1c748 100644 --- a/ScoutSuite/providers/azure/facade/facade.py +++ b/ScoutSuite/providers/azure/facade/facade.py @@ -28,4 +28,4 @@ def __init__(self, credentials: AzureCredentials): self.loadbalancer = LoadBalancerFacade(credentials.credentials, credentials.subscription_id) self.rediscache = RedisCacheFacade(credentials.credentials, credentials.subscription_id) except NameError: - pass \ No newline at end of file + pass From 7bc5644bd48dac19534682eef049beb55502589d Mon Sep 17 00:00:00 2001 From: misg Date: Sat, 30 Mar 2019 23:43:36 -0400 Subject: [PATCH 662/667] Make the resource fetching of all services concurrent. --- ScoutSuite/providers/base/configs/services.py | 59 +++++++++++-------- 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/ScoutSuite/providers/base/configs/services.py b/ScoutSuite/providers/base/configs/services.py index 3ab416935..5aa6cefdf 100644 --- a/ScoutSuite/providers/base/configs/services.py +++ b/ScoutSuite/providers/base/configs/services.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +import asyncio from ScoutSuite.utils import format_service_name from ScoutSuite.core.console import print_error, print_exception, print_debug, print_info @@ -16,28 +16,35 @@ def _is_provider(self, provider_name): async def fetch(self, services=None, regions=None): services = [] if services is None else services regions = [] if regions is None else regions - for service in vars(self): - try: - # skip services - if services != [] and service not in services: - print_debug('Skipping the {} service'.format(format_service_name(service))) - continue - - print_info('Fetching resources for the {} service'.format(format_service_name(service))) - service_config = getattr(self, service) - # call fetch method for the service - if 'fetch_all' in dir(service_config): - method_args = {'credentials': self.credentials, 'regions': regions} - - if self._is_provider('aws'): - if service != 'iam': - method_args['partition_name'] = get_partition_name(self.credentials) - - await service_config.fetch_all(**method_args) - if hasattr(service_config, 'finalize'): - await service_config.finalize() - else: - print_debug('No method to fetch service %s.' % service) - except Exception as e: - print_error('Error: could not fetch %s configuration.' % service) - print_exception(e) + tasks = { + asyncio.ensure_future( + self._fetch(service, services, regions) + ) for service in vars(self) + } + await asyncio.wait(tasks) + + async def _fetch(self, service, services, regions): + try: + # skip services + if services != [] and service not in services: + print_debug('Skipping the {} service'.format(format_service_name(service))) + return + + print_info('Fetching resources for the {} service'.format(format_service_name(service))) + service_config = getattr(self, service) + # call fetch method for the service + if 'fetch_all' in dir(service_config): + method_args = {'credentials': self.credentials, 'regions': regions} + + if self._is_provider('aws'): + if service != 'iam': + method_args['partition_name'] = get_partition_name(self.credentials) + + await service_config.fetch_all(**method_args) + if hasattr(service_config, 'finalize'): + await service_config.finalize() + else: + print_debug('No method to fetch service %s.' % service) + except Exception as e: + print_error('Error: could not fetch %s configuration.' % service) + print_exception(e) From a8a98ad59d43295910514dda5d08eb970c0079f2 Mon Sep 17 00:00:00 2001 From: misg Date: Sun, 31 Mar 2019 11:18:01 -0400 Subject: [PATCH 663/667] Handle skipped services in fetch method instead of _fetch. --- ScoutSuite/providers/base/configs/services.py | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/ScoutSuite/providers/base/configs/services.py b/ScoutSuite/providers/base/configs/services.py index 5aa6cefdf..3c350e027 100644 --- a/ScoutSuite/providers/base/configs/services.py +++ b/ScoutSuite/providers/base/configs/services.py @@ -14,22 +14,25 @@ def _is_provider(self, provider_name): return False async def fetch(self, services=None, regions=None): - services = [] if services is None else services + # If services is set to None, fetch all services: + services = vars(self) if services is None else services regions = [] if regions is None else regions + + # First, print services that are going to get skipped: + for service in vars(self): + if service not in services: + print_debug('Skipping the {} service'.format(format_service_name(service))) + + # Then, fetch concurrently all services: tasks = { asyncio.ensure_future( - self._fetch(service, services, regions) - ) for service in vars(self) + self._fetch(service, regions) + ) for service in services } await asyncio.wait(tasks) - async def _fetch(self, service, services, regions): + async def _fetch(self, service, regions): try: - # skip services - if services != [] and service not in services: - print_debug('Skipping the {} service'.format(format_service_name(service))) - return - print_info('Fetching resources for the {} service'.format(format_service_name(service))) service_config = getattr(self, service) # call fetch method for the service From 2719064306687029f99964a6f44b2f2ee451d0a5 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 1 Apr 2019 15:53:28 -0400 Subject: [PATCH 664/667] Added exception --- .../providers/base/authentication_strategy_factory.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ScoutSuite/providers/base/authentication_strategy_factory.py b/ScoutSuite/providers/base/authentication_strategy_factory.py index c79da9f83..d851ea48c 100644 --- a/ScoutSuite/providers/base/authentication_strategy_factory.py +++ b/ScoutSuite/providers/base/authentication_strategy_factory.py @@ -9,5 +9,11 @@ } +class InvalidAuthenticationStrategyException(Exception): pass + + def get_authentication_strategy(provider: str): + if provider not in _strategies: + raise InvalidAuthenticationStrategyException() + return _strategies[provider]() From df5ed0f3ccf5c9bcc2eee1e5a085e44ce1832088 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 1 Apr 2019 15:53:51 -0400 Subject: [PATCH 665/667] Upped version --- ScoutSuite/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/__init__.py b/ScoutSuite/__init__.py index 66e4dd7f4..84400fa7c 100644 --- a/ScoutSuite/__init__.py +++ b/ScoutSuite/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- __author__ = 'NCC Group' -__version__ = '4.3.0' +__version__ = '5.0.0-alpha' DEFAULT_REPORT_DIR = 'scoutsuite-report' DEFAULT_RESULT_FILE = 'scoutsuite-results/scoutsuite_results.js' From d634d08949a851a450bd124353d6a5f2f951b3a6 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 1 Apr 2019 16:14:03 -0400 Subject: [PATCH 666/667] Initialised variable do None --- ScoutSuite/__main__.py | 1 + ScoutSuite/providers/aws/provider.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/__main__.py b/ScoutSuite/__main__.py index ff75846f1..d3c9b17b2 100644 --- a/ScoutSuite/__main__.py +++ b/ScoutSuite/__main__.py @@ -42,6 +42,7 @@ async def run_scan(args): print_info('Launching Scout') + credentials = None if not args.get('fetch_local'): auth_strategy = get_authentication_strategy(args.get('provider')) credentials = auth_strategy.authenticate(profile=args.get('profile'), diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index 0db16dbdc..37ad18040 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -36,7 +36,8 @@ def __init__(self, profile='default', report_dir=None, timestamp=None, services= self.provider_name = 'Amazon Web Services' self.credentials = kwargs['credentials'] - self.aws_account_id = get_aws_account_id(self.credentials) + if self.credentials: + self.aws_account_id = get_aws_account_id(self.credentials) super(AWSProvider, self).__init__(report_dir, timestamp, services, skipped_services, thread_config) From 1c5bc5099af2b7fe15488318980b655fcc04fe93 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Mon, 1 Apr 2019 16:18:40 -0400 Subject: [PATCH 667/667] Fixed credentials none on local runs --- ScoutSuite/providers/azure/provider.py | 3 ++- ScoutSuite/providers/gcp/provider.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/providers/azure/provider.py b/ScoutSuite/providers/azure/provider.py index da22e6099..f2cbaeed3 100644 --- a/ScoutSuite/providers/azure/provider.py +++ b/ScoutSuite/providers/azure/provider.py @@ -29,7 +29,8 @@ def __init__(self, project_id=None, organization_id=None, self.services_config = AzureServicesConfig self.credentials = kwargs['credentials'] - self.aws_account_id = self.credentials.aws_account_id # TODO : Get rid of aws_account_id + if self.credentials: + self.aws_account_id = self.credentials.aws_account_id # TODO : Get rid of aws_account_id super(AzureProvider, self).__init__(report_dir, timestamp, services, skipped_services, thread_config) diff --git a/ScoutSuite/providers/gcp/provider.py b/ScoutSuite/providers/gcp/provider.py index 75980bbd3..3fb8608f2 100644 --- a/ScoutSuite/providers/gcp/provider.py +++ b/ScoutSuite/providers/gcp/provider.py @@ -38,7 +38,8 @@ def __init__(self, project_id=None, folder_id=None, organization_id=None, all_pr self.services_config = GCPServicesConfig self.credentials = kwargs['credentials'] - self._set_aws_account_id() + if self.credentials: + self._set_aws_account_id() super(GCPProvider, self).__init__(report_dir, timestamp, services, skipped_services, thread_config)
            ListUpload/DeleteView PermissionsEdit Permissions
            ListUpload/DeleteView
            Permissions            		Edit
            Permissions
            {{DisplayName}}{{DisplayName}} {{{s3_grant_2_icon permissions.read}}}