Merge pull request #806 from nccgroup/develop

release/5.10

.coveragerc

@@ -3,11 +3,11 @@ include =   ScoutSuite/*.py
             ScoutSuite/core/*.py
             ScoutSuite/output/*.py
             ScoutSuite/providers/aws/*.py
-            ScoutSuite/providers/aws/configs/*.py
-            ScoutSuite/providers/aws/services/*.py
+            ScoutSuite/providers/aws/facade/*.py
+            ScoutSuite/providers/aws/resources/*.py
             ScoutSuite/providers/azure/*.py
-            ScoutSuite/providers/azure/configs/*.py
-            ScoutSuite/providers/azure/services/*.py
+            ScoutSuite/providers/azure/facade/*.py
+            ScoutSuite/providers/azure/resources/*.py
             ScoutSuite/providers/gcp/*.py
-            ScoutSuite/providers/gcp/configs/*.py
-            ScoutSuite/providers/gcp/services/*.py
+            ScoutSuite/providers/gcp/facade/*.py
+            ScoutSuite/providers/gcp/resources/*.py

README.md

@@ -4,12 +4,12 @@
 
 #
 
-[![Travis](https://travis-ci.org/nccgroup/ScoutSuite.svg?branch=master)](https://travis-ci.org/nccgroup/ScoutSuite)
-[![Coverage Status](https://coveralls.io/repos/github/nccgroup/ScoutSuite/badge.svg?branch=master)](https://coveralls.io/github/nccgroup/ScoutSuite?branch=master)
+[![Workflow](https://github.com/nccgroup/ScoutSuite/workflows/CI%20Workflow/badge.svg)](https://github.com/nccgroup/ScoutSuite/actions)
 [![CodeCov](https://codecov.io/gh/nccgroup/ScoutSuite/branch/master/graph/badge.svg)](https://codecov.io/gh/nccgroup/ScoutSuite)
-[![Total alerts](https://img.shields.io/lgtm/alerts/g/nccgroup/ScoutSuite.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/nccgroup/ScoutSuite/alerts/)
 [![PyPI version](https://badge.fury.io/py/ScoutSuite.svg)](https://badge.fury.io/py/ScoutSuite)
 [![PyPI downloads](https://img.shields.io/pypi/dm/scoutsuite)](https://img.shields.io/pypi/dm/scoutsuite)
+[![Docker Hub](https://img.shields.io/badge/Docker%20Hub-rossja%2Fncc--scoutsuite-blue)](https://hub.docker.com/r/rossja/ncc-scoutsuite/)
+[![Docker Pulls](https://img.shields.io/docker/pulls/rossja/ncc-scoutsuite.svg?style=flat-square)](https://hub.docker.com/r/rossja/ncc-scoutsuite/)
 
 ## Description
 

ScoutSuite/__init__.py

@@ -1,5 +1,5 @@
 __author__ = 'NCC Group'
-__version__ = '5.9.1'
+__version__ = '5.10.0'
 
 ERRORS_LIST = []
 

ScoutSuite/__main__.py

@@ -204,7 +204,7 @@ async def _run(provider,
         if not credentials:
             return 101
     except Exception as e:
-        print_exception('Authentication failure: {}'.format(e))
+        print_exception(f'Authentication failure: {e}')
         return 101
 
     # Create a cloud provider object
@@ -228,7 +228,7 @@ async def _run(provider,
                                       programmatic_execution=programmatic_execution,
                                       credentials=credentials)
     except Exception as e:
-        print_exception('Initialization failure: {}'.format(e))
+        print_exception(f'Initialization failure: {e}')
         return 102
 
     # Create a new report
@@ -336,7 +336,7 @@ async def _run(provider,
             exceptions.process(cloud_provider)
             exceptions = exceptions.exceptions
         except Exception as e:
-            print_exception('Failed to load exceptions: {}'.format(e))
+            print_exception(f'Failed to load exceptions: {e}')
             exceptions = {}
     else:
         exceptions = {}

ScoutSuite/core/cli_parser.py

@@ -15,7 +15,7 @@ def __init__(self):
 
         self.parser.add_argument('-v', '--version',
                                  action='version',
-                                 version='Scout Suite {}'.format(__version__))
+                                 version=f'Scout Suite {__version__}')
 
         self.common_providers_args_parser = argparse.ArgumentParser(add_help=False)
 

ScoutSuite/core/conditions.py

@@ -210,6 +210,29 @@ def pass_condition(b, test, a):
     elif test == 'notInSubnets':
         result = (not pass_condition(b, 'inSubnets', a))
 
+    # Port/port ranges tests
+    elif test == 'portsInPortList':
+        result = False
+        if not type(b) == list:
+            b = [b]
+        if not type(a) == list:
+            a = [a]
+        for port_range in b:
+            if '-' in port_range:
+                bottom_limit_port = int(port_range.split('-')[0])
+                upper_limit_port = int(port_range.split('-')[1])
+                for port in a:
+                    if type(port) != int:
+                        port = int(port)
+                    if bottom_limit_port <= port <= upper_limit_port:
+                        result = True
+                        break
+            else: #A single port
+                for port in a:
+                    if port == port_range:
+                        result = True
+                        break
+
     # Policy statement tests
     elif test == 'containAction':
         result = False

ScoutSuite/core/console.py

@@ -1,10 +1,10 @@
 import logging
+import platform
 import os
 import sys
 import traceback
 
 import coloredlogs
-from six.moves import input
 
 from ScoutSuite import ERRORS_LIST
 
@@ -32,8 +32,18 @@ def set_logger_configuration(is_debug=False, quiet=False, output_file_path=None)
         coloredlogs.install(level='DEBUG' if is_debug else 'INFO', logger=logger)
 
     if output_file_path:
+        # For some reason, hostname information is not passed to the FileHandler
+        # Add it using a filter
+        class HostnameFilter(logging.Filter):
+            hostname = platform.node()
+
+            def filter(self, record):
+                record.hostname = HostnameFilter.hostname
+                return True
         # create file handler which logs messages
         fh = logging.FileHandler(output_file_path, 'w+')
+        # Add filter to add hostname information
+        fh.addFilter(HostnameFilter())
         # create formatter and add it to the handlers
         formatter = logging.Formatter(fmt='%(asctime)s %(hostname)s %(name)s[%(process)d] %(levelname)s %(message)s',
                                       datefmt='%Y-%m-%d %H:%M:%S')
@@ -66,18 +76,18 @@ def print_exception(exception, additional_details=None):
             file_name = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
             line_number = exc_tb.tb_lineno
             traceback_exc = traceback.format_exc()
-            str = '{} L{}: {}'.format(file_name, line_number, exception)
+            str = f'{file_name} L{line_number}: {exception}'
         else:
             file_name = None
             line_number = None
             traceback_exc = None
-            str = '{}'.format(exception)
+            str = f'{exception}'
             exc = False  # if there isn't an actual exception then it's pointless
     except Exception as e:
         file_name = None
         line_number = None
         traceback_exc = None
-        str = '{}'.format(exception)
+        str = f'{exception}'
 
     if verbose_exceptions and exc:
         logger.exception(str)
@@ -86,8 +96,8 @@ def print_exception(exception, additional_details=None):
 
     ERRORS_LIST.append({'file': file_name,
                         'line': line_number,
-                        'exception': '{}'.format(exception),
-                        'traceback': '{}'.format(traceback_exc),
+                        'exception': f'{exception}',
+                        'traceback': f'{traceback_exc}',
                         'additional_details': additional_details})
 
 
@@ -131,7 +141,7 @@ def prompt_overwrite(filename, force_write, test_input=None):
     """
     if not os.path.exists(filename) or force_write:
         return True
-    return prompt_yes_no('File \'{}\' already exists. Do you want to overwrite it'.format(filename),
+    return prompt_yes_no(f'File \'{filename}\' already exists. Do you want to overwrite it',
                          test_input=test_input)
 
 

ScoutSuite/core/exceptions.py

@@ -3,7 +3,7 @@
 from ScoutSuite.output.result_encoder import JavaScriptEncoder
 
 
-class RuleExceptions(object):
+class RuleExceptions:
     """
     Exceptions handling
     """

ScoutSuite/core/fs.py

@@ -1,5 +1,3 @@
-from __future__ import print_function
-
 import datetime
 import json
 import os

ScoutSuite/core/processingengine.py

@@ -4,7 +4,7 @@
 from ScoutSuite.core.utils import recurse
 
 
-class ProcessingEngine(object):
+class ProcessingEngine:
     """
 
     """
@@ -21,7 +21,7 @@ def __init__(self, ruleset):
                     manage_dictionary(self.rules, rule.path, [])
                     self.rules[rule.path].append(rule)
                 except Exception as e:
-                    print_exception('Failed to create rule %s: %s' % (rule.filename, e))
+                    print_exception(f'Failed to create rule {rule.filename}: {e}')
 
     def run(self, cloud_provider, skip_dashboard=False):
         # Clean up existing findings
@@ -35,7 +35,7 @@ def run(self, cloud_provider, skip_dashboard=False):
                 if not rule.enabled:  # or rule.service not in []: # TODO: handle this...
                     continue
 
-                print_debug('Processing %s rule "%s" (%s)' % (rule.service, rule.description, rule.filename))
+                print_debug(f'Processing {rule.service} rule "{rule.description}" ({rule.filename})')
                 finding_path = rule.path
                 path = finding_path.split('.')
                 service = path[0]
@@ -68,7 +68,7 @@ def run(self, cloud_provider, skip_dashboard=False):
                     cloud_provider.services[service][self.ruleset.rule_type][rule.key]['references'] = \
                         rule.references if hasattr(rule, 'references') else None
                 except Exception as e:
-                    print_exception('Failed to process rule defined in %s: %s' % (rule.filename, e))
+                    print_exception(f'Failed to process rule defined in {rule.filename}: {e}')
                     # Fallback if process rule failed to ensure report creation and data dump still happen
                     cloud_provider.services[service][self.ruleset.rule_type][rule.key]['checked_items'] = 0
                     cloud_provider.services[service][self.ruleset.rule_type][rule.key]['flagged_items'] = 0

ScoutSuite/core/rule.py

@@ -29,7 +29,7 @@
 ]
 
 
-class Rule(object):
+class Rule:
 
     def to_string(self):
         return str(vars(self))
@@ -71,8 +71,8 @@ def set_definition(self, rule_definitions, attributes=None, ip_ranges=None, para
                 if condition[0].startswith('_INCLUDE_('):
                     include = re.findall(r'_INCLUDE_\((.*?)\)', condition[0])[0]
                     # new_conditions = load_data(include, key_name = 'conditions')
-                    rules_path = '%s/%s' % (self.data_path, include)
-                    with open(rules_path, 'rt') as f:
+                    rules_path = f'{self.data_path}/{include}'
+                    with open(rules_path) as f:
                         new_conditions = f.read()
                         for (i, value) in enumerate(condition[1]):
                             new_conditions = re.sub(condition[1][i], condition[2][i], new_conditions)
@@ -139,6 +139,6 @@ def set_definition(self, rule_definitions, attributes=None, ip_ranges=None, para
                 setattr(self, 'key', self.filename)
             setattr(self, 'key', self.key.replace('.json', ''))
             if self.key_suffix:
-                setattr(self, 'key', '%s-%s' % (self.key, self.key_suffix))
+                setattr(self, 'key', f'{self.key}-{self.key_suffix}')
         except Exception as e:
-            print_exception('Failed to set definition %s: %s' % (self.filename, e))
+            print_exception(f'Failed to set definition {self.filename}: {e}')

ScoutSuite/core/rule_definition.py

@@ -4,7 +4,7 @@
 from ScoutSuite.core.console import print_error, print_exception
 
 
-class RuleDefinition(object):
+class RuleDefinition:
 
     def __init__(self, data_path, file_name=None, rule_dirs=None, string_definition=None):
         rule_dirs = [] if rule_dirs is None else rule_dirs
@@ -28,7 +28,7 @@ def __str__(self):
         value = '-' * 80 + '\n' + ' ' * padding + ' %s' % getattr(self, 'description') + '\n' + '-' * 80 + '\n'
         quiet_list = ['descriptions', 'rule_dirs', 'rule_types', 'rules_data_path', 'string_definition']
         value += '\n'.join(
-            '%s: %s' % (attr, str(getattr(self, attr))) for attr in vars(self) if attr not in quiet_list)
+            '{}: {}'.format(attr, str(getattr(self, attr))) for attr in vars(self) if attr not in quiet_list)
         value += '\n'
         return value
 
@@ -46,7 +46,7 @@ def load(self):
             try:
                 file_path = os.path.join(rule_dir, self.file_name) if rule_dir else self.file_name
             except Exception as e:
-                print_exception('Failed to load file %s: %s' % (self.file_name, str(e)))
+                print_exception('Failed to load file {}: {}'.format(self.file_name, str(e)))
             if os.path.isfile(file_path):
                 self.file_path = file_path
                 file_name_valid = True
@@ -72,16 +72,16 @@ def load(self):
             print_error('Error: could not find %s' % self.file_name)
         else:
             try:
-                with open(self.file_path, 'rt') as f:
+                with open(self.file_path) as f:
                     self.string_definition = f.read()
                     self.load_from_string_definition()
             except Exception as e:
-                print_exception('Failed to load rule defined in %s: %s' % (self.file_name, str(e)))
+                print_exception('Failed to load rule defined in {}: {}'.format(self.file_name, str(e)))
 
     def load_from_string_definition(self):
         try:
             definition = json.loads(self.string_definition)
             for attr in definition:
                 setattr(self, attr, definition[attr])
         except Exception as e:
-            print_exception('Failed to load string definition %s: %s' % (self.string_definition, str(e)))
+            print_exception('Failed to load string definition {}: {}'.format(self.string_definition, str(e)))

ScoutSuite/core/ruleset.py

@@ -73,7 +73,7 @@ def load(self, rule_type, quiet=False):
         """
         if self.filename and os.path.exists(self.filename):
             try:
-                with open(self.filename, 'rt') as f:
+                with open(self.filename) as f:
                     ruleset = json.load(f)
                     self.about = ruleset['about'] if 'about' in ruleset else ''
                     self.rules = {}
@@ -82,7 +82,7 @@ def load(self, rule_type, quiet=False):
                         for rule in ruleset['rules'][filename]:
                             self.handle_rule_versions(filename, rule_type, rule)
             except Exception as e:
-                print_exception('Ruleset file %s contains malformed JSON: %s' % (self.filename, e))
+                print_exception(f'Ruleset file {self.filename} contains malformed JSON: {e}')
                 self.rules = []
                 self.about = ''
         else:
@@ -193,7 +193,7 @@ def find_file(self, filename, filetype='rulesets'):
         if filename and not os.path.isfile(filename):
             # Not a valid relative / absolute path, check Scout's data under findings/ or filters/
             if not filename.startswith('findings/') and not filename.startswith('filters/'):
-                filename = '%s/%s' % (filetype, filename)
+                filename = f'{filetype}/{filename}'
             if not os.path.isfile(filename):
                 filename = os.path.join(self.rules_data_path, filename)
             if not os.path.isfile(filename) and not filename.endswith('.json'):

ScoutSuite/core/server.py

@@ -7,7 +7,7 @@
 count_re = re.compile(r".*_count$")
 
 
-class Server(object):
+class Server:
     """
     Boots a server that serves the result of the report for the user. This is still a proof of concept,
     but will eventually be used to serve data when it exceeds 400mb.

ScoutSuite/core/utils.py

@@ -2,7 +2,6 @@
 Single-service rule processing functions
 """
 
-from six import string_types
 import copy
 
 from ScoutSuite.core.console import print_exception
@@ -74,7 +73,7 @@ def recurse(all_info, current_info, target_path, current_path, config, add_suffi
             results = results + recurse(all_info, split_current_info, copy.deepcopy(target_path), split_current_path,
                                         config, add_suffix)
     # Python 2-3 compatible way to check for string type
-    elif isinstance(current_info, string_types):
+    elif isinstance(current_info, str):
         split_current_path = copy.deepcopy(current_path)
         results = results + recurse(all_info, current_info, [], split_current_path,
                                     config, add_suffix)