diff --git a/README.md b/README.md index af39d46fea..0e507e7137 100644 --- a/README.md +++ b/README.md @@ -166,6 +166,7 @@ Optional parameters: | -u, --update-specs | | | Update the [CloudFormation Resource Specifications](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cfn-resource-specification.html). You may need sudo to run this. You will need internet access when running this command | | -o, --override-spec | | filename | Spec-style file containing custom definitions. Can be used to override CloudFormation specifications. More info [here](#customize-specifications) | | -g, --build-graph | | | Creates a file in the same directory as the template that models the template's resources in [DOT format](https://en.wikipedia.org/wiki/DOT_(graph_description_language)) | +| -s, --registry-schemas | | | one or more directories of [CloudFormation Registry](https://aws.amazon.com/blogs/aws/cloudformation-update-cli-third-party-resource-support-registry/) [Resource Schemas](https://github.com/aws-cloudformation/aws-cloudformation-resource-schema/) | -v, --version | | | Version of cfn-lint | ### Info Rules diff --git a/src/cfnlint/__main__.py b/src/cfnlint/__main__.py index 0af2fb38d7..dfb7c59a23 100644 --- a/src/cfnlint/__main__.py +++ b/src/cfnlint/__main__.py @@ -12,7 +12,6 @@ def main(): - """Main function""" if sys.version_info[:2] == (3, 4): warnings.warn('Python 3.4 has reached end of life. ' 'cfn-lint will end support for python 3.4 on July 1st, 2020.', Warning, stacklevel=3) @@ -33,7 +32,7 @@ def main(): matches.extend( cfnlint.core.run_cli( filename, template, rules, - args.regions, args.override_spec, args.build_graph, args.mandatory_checks)) + args.regions, args.override_spec, args.build_graph, args.registry_schemas, args.mandatory_checks)) else: matches.extend(errors) LOGGER.debug('Completed linting of file: %s', str(filename)) diff --git a/src/cfnlint/conditions.py b/src/cfnlint/conditions.py index 66f2c75dc4..03e97b68c5 100644 --- a/src/cfnlint/conditions.py +++ b/src/cfnlint/conditions.py @@ -138,7 +138,6 @@ def process_influenced_equal(self, equal): self.Influenced_Equals[equal.Right.Function].add(equal.Left.String) def process_condition(self, template, value): - """ process condition """ if isinstance(value, dict): if len(value) == 1: for func_name, func_value in value.items(): @@ -166,7 +165,6 @@ def process_condition(self, template, value): raise ConditionParseError def process_function(self, template, values): - """ Process Function """ results = [] for value in values: if isinstance(value, dict): diff --git a/src/cfnlint/config.py b/src/cfnlint/config.py index 8ec5602eac..13272ba877 100644 --- a/src/cfnlint/config.py +++ b/src/cfnlint/config.py @@ -23,7 +23,6 @@ def configure_logging(debug_logging, info_logging): - """Setup Logging""" ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) @@ -333,7 +332,7 @@ def __call__(self, parser, namespace, values, option_string=None): standard = parser.add_argument_group('Standard') advanced = parser.add_argument_group('Advanced / Debugging') - # Alllow the template to be passes as an optional or a positional argument + # Allow the template to be passes as an optional or a positional argument standard.add_argument( 'templates', metavar='TEMPLATE', nargs='*', help='The CloudFormation template to be linted') standard.add_argument( @@ -356,7 +355,6 @@ def __call__(self, parser, namespace, values, option_string=None): standard.add_argument( '-f', '--format', help='Output Format', choices=['quiet', 'parseable', 'json', 'junit', 'pretty'] ) - standard.add_argument( '-l', '--list-rules', dest='listrules', default=False, action='store_true', help='list all the rules' @@ -390,25 +388,23 @@ def __call__(self, parser, namespace, values, option_string=None): standard.add_argument( '-e', '--include-experimental', help='Include experimental rules', action='store_true' ) - standard.add_argument( '-x', '--configure-rule', dest='configure_rules', nargs='+', default={}, action=RuleConfigurationAction, help='Provide configuration for a rule. Format RuleId:key=value. Example: E3012:strict=false' ) - standard.add_argument('--config-file', dest='config_file', help='Specify the cfnlintrc file to use') - advanced.add_argument( '-o', '--override-spec', dest='override_spec', help='A CloudFormation Spec override file that allows customization' ) - advanced.add_argument( '-g', '--build-graph', help='Creates a file in the same directory as the template that models the template\'s resources in DOT format', action='store_true' ) - + advanced.add_argument( + '-s', '--registry-schemas', help='one or more directories of CloudFormation Registry Schemas', action='extend', type=comma_separated_arg, nargs='+' + ) standard.add_argument( '-v', '--version', help='Version of cfn-lint', action='version', version='%(prog)s {version}'.format(version=__version__) @@ -425,7 +421,6 @@ def __call__(self, parser, namespace, values, option_string=None): '--update-iam-policies', help=argparse.SUPPRESS, action='store_true' ) - standard.add_argument( '--output-file', type=str, default=None, help='Writes the output to the specified file, ideal for producing reports' @@ -441,11 +436,9 @@ def __init__(self, template_args): self.set_template_args(template_args) def get_template_args(self): - """ Get Template Args""" return self._template_args def set_template_args(self, template): - """ Set Template Args""" defaults = {} if isinstance(template, dict): configs = template.get('Metadata', {}).get('cfn-lint', {}).get('config', {}) @@ -491,7 +484,6 @@ def __init__(self, cli_args): self, config_file=self._get_argument_value('config_file', False, False)) def _get_argument_value(self, arg_name, is_template, is_config_file): - """ Get Argument value """ cli_value = getattr(self.cli_args, arg_name) template_value = self.template_args.get(arg_name) file_value = self.file_args.get(arg_name) @@ -505,28 +497,23 @@ def _get_argument_value(self, arg_name, is_template, is_config_file): @property def ignore_checks(self): - """ ignore_checks """ return self._get_argument_value('ignore_checks', True, True) @property def include_checks(self): - """ include_checks """ results = self._get_argument_value('include_checks', True, True) return ['W', 'E'] + results @property def mandatory_checks(self): - """ mandatory_checks """ return self._get_argument_value('mandatory_checks', False, True) @property def include_experimental(self): - """ include_experimental """ return self._get_argument_value('include_experimental', True, True) @property def regions(self): - """ regions """ results = self._get_argument_value('regions', True, True) if not results: return ['us-east-1'] @@ -536,22 +523,18 @@ def regions(self): @property def ignore_bad_template(self): - """ ignore_bad_template """ return self._get_argument_value('ignore_bad_template', True, True) @property def debug(self): - """ debug """ return self._get_argument_value('debug', False, False) @property def format(self): - """ format """ return self._get_argument_value('format', False, True) @property def templates(self): - """ templates """ templates_args = self._get_argument_value('templates', False, True) template_alt_args = self._get_argument_value('template_alt', False, False) if template_alt_args: @@ -588,7 +571,6 @@ def templates(self): return sorted(all_filenames) def _ignore_templates(self): - """ templates """ ignore_template_args = self._get_argument_value('ignore_templates', False, True) if ignore_template_args: filenames = ignore_template_args @@ -619,50 +601,44 @@ def _ignore_templates(self): @property def append_rules(self): - """ append_rules """ return self._get_argument_value('append_rules', False, True) @property def override_spec(self): - """ override_spec """ return self._get_argument_value('override_spec', False, True) @property def update_specs(self): - """ update_specs """ return self._get_argument_value('update_specs', False, False) @property def update_documentation(self): - """ update_specs """ return self._get_argument_value('update_documentation', False, False) @property def update_iam_policies(self): - """ update_iam_policies """ return self._get_argument_value('update_iam_policies', False, False) @property def listrules(self): - """ listrules """ return self._get_argument_value('listrules', False, False) @property def configure_rules(self): - """ Configure rules """ return self._get_argument_value('configure_rules', True, True) @property def config_file(self): - """ Config file """ return self._get_argument_value('config_file', False, False) @property def build_graph(self): - """ build_graph """ return self._get_argument_value('build_graph', False, False) @property def output_file(self): - """ output_file """ return self._get_argument_value('output_file', False, True) + + @property + def registry_schemas(self): + return self._get_argument_value('registry_schemas', False, True) diff --git a/src/cfnlint/core.py b/src/cfnlint/core.py index 02d752b428..a7c953b991 100644 --- a/src/cfnlint/core.py +++ b/src/cfnlint/core.py @@ -2,6 +2,7 @@ Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. SPDX-License-Identifier: MIT-0 """ +import json import logging import os import sys @@ -15,7 +16,7 @@ import cfnlint.formatters import cfnlint.decode import cfnlint.maintenance -from cfnlint.helpers import REGIONS +from cfnlint.helpers import REGIONS, REGISTRY_SCHEMAS LOGGER = logging.getLogger('cfnlint') DEFAULT_RULESDIR = os.path.join(os.path.dirname(__file__), 'rules') @@ -39,7 +40,7 @@ class UnexpectedRuleException(CfnLintExitException): """When processing a rule fails in an unexpected way""" -def run_cli(filename, template, rules, regions, override_spec, build_graph, mandatory_rules=None): +def run_cli(filename, template, rules, regions, override_spec, build_graph, registry_schemas, mandatory_rules=None): """Process args and run""" if override_spec: @@ -49,6 +50,13 @@ def run_cli(filename, template, rules, regions, override_spec, build_graph, mand template_obj = Template(filename, template, regions) template_obj.build_graph() + if registry_schemas: + for path in registry_schemas: + if path and os.path.isdir(os.path.expanduser(path)): + for f in os.listdir(path): + with open(os.path.join(path, f)) as schema: + REGISTRY_SCHEMAS.append(json.load(schema)) + return run_checks(filename, template, rules, regions, mandatory_rules) @@ -67,7 +75,6 @@ def get_exit_code(matches): def get_formatter(fmt): - """ Get Formatter""" formatter = {} if fmt: if fmt == 'quiet': @@ -88,7 +95,6 @@ def get_formatter(fmt): def get_rules(append_rules, ignore_rules, include_rules, configure_rules=None, include_experimental=False, mandatory_rules=None): - """Get rules""" rules = RulesCollection(ignore_rules, include_rules, configure_rules, include_experimental, mandatory_rules) rules_paths = [DEFAULT_RULESDIR] + append_rules diff --git a/src/cfnlint/helpers.py b/src/cfnlint/helpers.py index 3c6bf086ff..d542fbe6f5 100644 --- a/src/cfnlint/helpers.py +++ b/src/cfnlint/helpers.py @@ -313,7 +313,7 @@ def load_resource(package, filename='us-east-1.json'): RESOURCE_SPECS = {} - +REGISTRY_SCHEMAS = [] def merge_spec(source, destination): """ Recursive merge spec dict """ diff --git a/src/cfnlint/maintenance.py b/src/cfnlint/maintenance.py index 034f4c7175..3760fa9c02 100644 --- a/src/cfnlint/maintenance.py +++ b/src/cfnlint/maintenance.py @@ -20,8 +20,6 @@ def update_resource_specs(): - """ Update Resource Specs """ - # Pool() uses cpu count if no number of processors is specified # Pool() only implements the Context Manager protocol from Python3.3 onwards, # so it will fail Python2.7 style linting, as well as throw AttributeError @@ -87,8 +85,6 @@ def search_and_replace_botocore_types(obj): json.dump(spec, f, indent=2, sort_keys=True, separators=(',', ': ')) def update_documentation(rules): - """Generate documentation""" - # Update the overview of all rules in the linter filename = 'docs/rules.md' diff --git a/src/cfnlint/rules/resources/Configuration.py b/src/cfnlint/rules/resources/Configuration.py index b1d215c584..6378f60d0d 100644 --- a/src/cfnlint/rules/resources/Configuration.py +++ b/src/cfnlint/rules/resources/Configuration.py @@ -3,6 +3,7 @@ SPDX-License-Identifier: MIT-0 """ import six +from cfnlint.helpers import REGISTRY_SCHEMAS from cfnlint.rules import CloudFormationLintRule from cfnlint.rules import RuleMatch import cfnlint.helpers @@ -99,7 +100,7 @@ def _check_resource(self, cfn, resource_name, resource_values): self.logger.debug('Check resource types by region...') for region, specs in cfnlint.helpers.RESOURCE_SPECS.items(): if region in cfn.regions: - if resource_type not in specs['ResourceTypes']: + if resource_type not in specs['ResourceTypes'] and resource_type not in [s['typeName'] for s in REGISTRY_SCHEMAS]: if not resource_type.startswith(('Custom::', 'AWS::Serverless::')) and not resource_type.endswith('::MODULE'): message = 'Invalid or unsupported Type {0} for resource {1} in {2}' matches.append(RuleMatch( diff --git a/src/cfnlint/rules/resources/ResourceSchema.py b/src/cfnlint/rules/resources/ResourceSchema.py new file mode 100644 index 0000000000..d61d575bf6 --- /dev/null +++ b/src/cfnlint/rules/resources/ResourceSchema.py @@ -0,0 +1,30 @@ +""" +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +SPDX-License-Identifier: MIT-0 +""" +import re +from jsonschema import validate, ValidationError +from cfnlint.helpers import REGEX_DYN_REF, PSEUDOPARAMS, FN_PREFIX, UNCONVERTED_SUFFIXES, REGISTRY_SCHEMAS +from cfnlint.rules import CloudFormationLintRule +from cfnlint.rules import RuleMatch + +class ResourceSchema(CloudFormationLintRule): + id = 'E3000' + shortdesc = 'Resource schema' + description = 'CloudFormation Registry resource schema validation' + source_url = 'https://github.com/aws-cloudformation/aws-cloudformation-resource-schema/' + tags = ['resources'] + + def match(self, cfn): + matches = [] + for schema in REGISTRY_SCHEMAS: + resource_type = schema['typeName'] + for resource_name, resource_values in cfn.get_resources([resource_type]).items(): + properties = resource_values.get('Properties', {}) + # ignoring resources with CloudFormation template syntax in Properties + if not re.match(REGEX_DYN_REF, str(properties)) and not any(x in str(properties) for x in PSEUDOPARAMS + UNCONVERTED_SUFFIXES) and FN_PREFIX not in str(properties): + try: + validate(properties, schema) + except ValidationError as e: + matches.append(RuleMatch(['Resources', resource_name, 'Properties'], e.message)) + return matches diff --git a/src/cfnlint/template.py b/src/cfnlint/template.py index c8718f1af3..87bbaf2522 100644 --- a/src/cfnlint/template.py +++ b/src/cfnlint/template.py @@ -81,7 +81,6 @@ def get_resources(self, resource_type=[]): return results def get_parameters(self): - """Get Resources""" LOGGER.debug('Get parameters from template...') parameters = self.template.get('Parameters', {}) if not parameters: @@ -105,7 +104,6 @@ def get_modules(self): return results def get_mappings(self): - """Get Resources""" LOGGER.debug('Get mapping from template...') mappings = self.template.get('Mappings', {}) if not mappings: @@ -114,7 +112,6 @@ def get_mappings(self): return mappings def get_resource_names(self): - """Get all the Resource Names""" LOGGER.debug('Get the names of all resources from template...') results = [] resources = self.template.get('Resources', {}) @@ -125,7 +122,6 @@ def get_resource_names(self): return results def get_parameter_names(self): - """Get all Parameter Names""" LOGGER.debug('Get names of all parameters from template...') results = [] parameters = self.template.get('Parameters', {}) @@ -136,7 +132,6 @@ def get_parameter_names(self): return results def get_valid_refs(self): - """Get all valid Refs""" LOGGER.debug('Get all valid REFs from template...') results = cfnlint.helpers.RegexDict() parameters = self.template.get('Parameters', {}) @@ -170,7 +165,6 @@ def get_valid_refs(self): return results def get_valid_getatts(self): - """Get all valid GetAtts""" LOGGER.debug('Get valid GetAtts from template...') resourcetypes = cfnlint.helpers.RESOURCE_SPECS['us-east-1'].get('ResourceTypes') results = {} @@ -206,7 +200,6 @@ def get_valid_getatts(self): return results def get_directives(self): - """ Get Directives""" results = {} for resource_name, resource_values in self.template.get('Resources', {}).items(): if isinstance(resource_values, dict): @@ -530,9 +523,6 @@ def check_value(self, obj, key, path, check_find_in_map=None, check_split=None, check_join=None, check_import_value=None, check_sub=None, **kwargs): - """ - Check the value - """ LOGGER.debug('Check value %s for %s', key, obj) matches = [] values_obj = self.get_values(obj=obj, key=key)