CloudFormation Registry resource schema support (--registry-schemas)

README.md

@@ -166,6 +166,7 @@ Optional parameters:
 | -u, --update-specs | | | Update the [CloudFormation Resource Specifications](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cfn-resource-specification.html).  You may need sudo to run this.  You will need internet access when running this command |
 | -o, --override-spec | | filename | Spec-style file containing custom definitions. Can be used to override CloudFormation specifications. More info [here](#customize-specifications) |
 | -g, --build-graph | |  | Creates a file in the same directory as the template that models the template's resources in [DOT format](https://en.wikipedia.org/wiki/DOT_(graph_description_language)) |
+| -s, --registry-schemas | | | one or more directories of [CloudFormation Registry](https://aws.amazon.com/blogs/aws/cloudformation-update-cli-third-party-resource-support-registry/) [Resource Schemas](https://github.com/aws-cloudformation/aws-cloudformation-resource-schema/)
 | -v, --version | | | Version of cfn-lint |
 
 ### Info Rules

src/cfnlint/__main__.py

@@ -12,7 +12,6 @@
 
 
 def main():
-    """Main function"""
     if sys.version_info[:2] == (3, 4):
         warnings.warn('Python 3.4 has reached end of life. '
                       'cfn-lint will end support for python 3.4 on July 1st, 2020.', Warning, stacklevel=3)
@@ -33,7 +32,7 @@ def main():
                 matches.extend(
                     cfnlint.core.run_cli(
                         filename, template, rules,
-                        args.regions, args.override_spec, args.build_graph, args.mandatory_checks))
+                        args.regions, args.override_spec, args.build_graph, args.registry_schemas, args.mandatory_checks))
             else:
                 matches.extend(errors)
             LOGGER.debug('Completed linting of file: %s', str(filename))

src/cfnlint/conditions.py

@@ -138,7 +138,6 @@ def process_influenced_equal(self, equal):
                 self.Influenced_Equals[equal.Right.Function].add(equal.Left.String)
 
     def process_condition(self, template, value):
-        """ process condition """
         if isinstance(value, dict):
             if len(value) == 1:
                 for func_name, func_value in value.items():
@@ -166,7 +165,6 @@ def process_condition(self, template, value):
             raise ConditionParseError
 
     def process_function(self, template, values):
-        """ Process Function """
         results = []
         for value in values:
             if isinstance(value, dict):

src/cfnlint/config.py

@@ -23,7 +23,6 @@
 
 
 def configure_logging(debug_logging, info_logging):
-    """Setup Logging"""
     ch = logging.StreamHandler()
     ch.setLevel(logging.DEBUG)
 
@@ -333,7 +332,7 @@ def __call__(self, parser, namespace, values, option_string=None):
         standard = parser.add_argument_group('Standard')
         advanced = parser.add_argument_group('Advanced / Debugging')
 
-        # Alllow the template to be passes as an optional or a positional argument
+        # Allow the template to be passes as an optional or a positional argument
         standard.add_argument(
             'templates', metavar='TEMPLATE', nargs='*', help='The CloudFormation template to be linted')
         standard.add_argument(
@@ -356,7 +355,6 @@ def __call__(self, parser, namespace, values, option_string=None):
         standard.add_argument(
             '-f', '--format', help='Output Format', choices=['quiet', 'parseable', 'json', 'junit', 'pretty']
         )
-
         standard.add_argument(
             '-l', '--list-rules', dest='listrules', default=False,
             action='store_true', help='list all the rules'
@@ -390,25 +388,23 @@ def __call__(self, parser, namespace, values, option_string=None):
         standard.add_argument(
             '-e', '--include-experimental', help='Include experimental rules', action='store_true'
         )
-
         standard.add_argument(
             '-x', '--configure-rule', dest='configure_rules', nargs='+', default={},
             action=RuleConfigurationAction,
             help='Provide configuration for a rule. Format RuleId:key=value. Example: E3012:strict=false'
         )
-
         standard.add_argument('--config-file', dest='config_file',
                               help='Specify the cfnlintrc file to use')
-
         advanced.add_argument(
             '-o', '--override-spec', dest='override_spec',
             help='A CloudFormation Spec override file that allows customization'
         )
-
         advanced.add_argument(
             '-g', '--build-graph', help='Creates a file in the same directory as the template that models the template\'s resources in DOT format', action='store_true'
         )
-
+        advanced.add_argument(
+            '-s', '--registry-schemas', help='one or more directories of CloudFormation Registry Schemas', action='extend', type=comma_separated_arg, nargs='+'
+        )
         standard.add_argument(
             '-v', '--version', help='Version of cfn-lint', action='version',
             version='%(prog)s {version}'.format(version=__version__)
@@ -425,7 +421,6 @@ def __call__(self, parser, namespace, values, option_string=None):
             '--update-iam-policies', help=argparse.SUPPRESS,
             action='store_true'
         )
-
         standard.add_argument(
             '--output-file', type=str, default=None,
             help='Writes the output to the specified file, ideal for producing reports'
@@ -441,11 +436,9 @@ def __init__(self, template_args):
         self.set_template_args(template_args)
 
     def get_template_args(self):
-        """ Get Template Args"""
         return self._template_args
 
     def set_template_args(self, template):
-        """ Set Template Args"""
         defaults = {}
         if isinstance(template, dict):
             configs = template.get('Metadata', {}).get('cfn-lint', {}).get('config', {})
@@ -491,7 +484,6 @@ def __init__(self, cli_args):
             self, config_file=self._get_argument_value('config_file', False, False))
 
     def _get_argument_value(self, arg_name, is_template, is_config_file):
-        """ Get Argument value """
         cli_value = getattr(self.cli_args, arg_name)
         template_value = self.template_args.get(arg_name)
         file_value = self.file_args.get(arg_name)
@@ -505,28 +497,23 @@ def _get_argument_value(self, arg_name, is_template, is_config_file):
 
     @property
     def ignore_checks(self):
-        """ ignore_checks """
         return self._get_argument_value('ignore_checks', True, True)
 
     @property
     def include_checks(self):
-        """ include_checks """
         results = self._get_argument_value('include_checks', True, True)
         return ['W', 'E'] + results
 
     @property
     def mandatory_checks(self):
-        """ mandatory_checks """
         return self._get_argument_value('mandatory_checks', False, True)
 
     @property
     def include_experimental(self):
-        """ include_experimental """
         return self._get_argument_value('include_experimental', True, True)
 
     @property
     def regions(self):
-        """ regions """
         results = self._get_argument_value('regions', True, True)
         if not results:
             return ['us-east-1']
@@ -536,22 +523,18 @@ def regions(self):
 
     @property
     def ignore_bad_template(self):
-        """ ignore_bad_template """
         return self._get_argument_value('ignore_bad_template', True, True)
 
     @property
     def debug(self):
-        """ debug """
         return self._get_argument_value('debug', False, False)
 
     @property
     def format(self):
-        """ format """
         return self._get_argument_value('format', False, True)
 
     @property
     def templates(self):
-        """ templates """
         templates_args = self._get_argument_value('templates', False, True)
         template_alt_args = self._get_argument_value('template_alt', False, False)
         if template_alt_args:
@@ -588,7 +571,6 @@ def templates(self):
         return sorted(all_filenames)
 
     def _ignore_templates(self):
-        """ templates """
         ignore_template_args = self._get_argument_value('ignore_templates', False, True)
         if ignore_template_args:
             filenames = ignore_template_args
@@ -619,50 +601,44 @@ def _ignore_templates(self):
 
     @property
     def append_rules(self):
-        """ append_rules """
         return self._get_argument_value('append_rules', False, True)
 
     @property
     def override_spec(self):
-        """ override_spec """
         return self._get_argument_value('override_spec', False, True)
 
     @property
     def update_specs(self):
-        """ update_specs """
         return self._get_argument_value('update_specs', False, False)
 
     @property
     def update_documentation(self):
-        """ update_specs """
         return self._get_argument_value('update_documentation', False, False)
 
     @property
     def update_iam_policies(self):
-        """ update_iam_policies """
         return self._get_argument_value('update_iam_policies', False, False)
 
     @property
     def listrules(self):
-        """ listrules """
         return self._get_argument_value('listrules', False, False)
 
     @property
     def configure_rules(self):
-        """ Configure rules """
         return self._get_argument_value('configure_rules', True, True)
 
     @property
     def config_file(self):
-        """ Config file """
         return self._get_argument_value('config_file', False, False)
 
     @property
     def build_graph(self):
-        """ build_graph """
         return self._get_argument_value('build_graph', False, False)
 
     @property
     def output_file(self):
-        """ output_file """
         return self._get_argument_value('output_file', False, True)
+
+    @property
+    def registry_schemas(self):
+        return self._get_argument_value('registry_schemas', False, True)

src/cfnlint/core.py

@@ -2,6 +2,7 @@
 Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 SPDX-License-Identifier: MIT-0
 """
+import json
 import logging
 import os
 import sys
@@ -15,7 +16,7 @@
 import cfnlint.formatters
 import cfnlint.decode
 import cfnlint.maintenance
-from cfnlint.helpers import REGIONS
+from cfnlint.helpers import REGIONS, REGISTRY_SCHEMAS
 
 LOGGER = logging.getLogger('cfnlint')
 DEFAULT_RULESDIR = os.path.join(os.path.dirname(__file__), 'rules')
@@ -39,7 +40,7 @@ class UnexpectedRuleException(CfnLintExitException):
     """When processing a rule fails in an unexpected way"""
 
 
-def run_cli(filename, template, rules, regions, override_spec, build_graph, mandatory_rules=None):
+def run_cli(filename, template, rules, regions, override_spec, build_graph, registry_schemas, mandatory_rules=None):
     """Process args and run"""
 
     if override_spec:
@@ -49,6 +50,13 @@ def run_cli(filename, template, rules, regions, override_spec, build_graph, mand
         template_obj = Template(filename, template, regions)
         template_obj.build_graph()
 
+    if registry_schemas:
+        for path in registry_schemas:
+            if path and os.path.isdir(os.path.expanduser(path)):
+                for f in os.listdir(path):
+                    with open(os.path.join(path, f)) as schema:
+                        REGISTRY_SCHEMAS.append(json.load(schema))
+
     return run_checks(filename, template, rules, regions, mandatory_rules)
 
 
@@ -67,7 +75,6 @@ def get_exit_code(matches):
 
 
 def get_formatter(fmt):
-    """ Get Formatter"""
     formatter = {}
     if fmt:
         if fmt == 'quiet':
@@ -88,7 +95,6 @@ def get_formatter(fmt):
 
 
 def get_rules(append_rules, ignore_rules, include_rules, configure_rules=None, include_experimental=False, mandatory_rules=None):
-    """Get rules"""
     rules = RulesCollection(ignore_rules, include_rules, configure_rules,
                             include_experimental, mandatory_rules)
     rules_paths = [DEFAULT_RULESDIR] + append_rules

src/cfnlint/helpers.py

@@ -313,7 +313,7 @@ def load_resource(package, filename='us-east-1.json'):
 
 
 RESOURCE_SPECS = {}
-
+REGISTRY_SCHEMAS = []
 
 def merge_spec(source, destination):
     """ Recursive merge spec dict """

src/cfnlint/maintenance.py

@@ -20,8 +20,6 @@
 
 
 def update_resource_specs():
-    """ Update Resource Specs """
-
     # Pool() uses cpu count if no number of processors is specified
     # Pool() only implements the Context Manager protocol from Python3.3 onwards,
     # so it will fail Python2.7 style linting, as well as throw AttributeError
@@ -87,8 +85,6 @@ def search_and_replace_botocore_types(obj):
         json.dump(spec, f, indent=2, sort_keys=True, separators=(',', ': '))
 
 def update_documentation(rules):
-    """Generate documentation"""
-
     # Update the overview of all rules in the linter
     filename = 'docs/rules.md'
 

src/cfnlint/rules/resources/Configuration.py

@@ -3,6 +3,7 @@
 SPDX-License-Identifier: MIT-0
 """
 import six
+from cfnlint.helpers import REGISTRY_SCHEMAS
 from cfnlint.rules import CloudFormationLintRule
 from cfnlint.rules import RuleMatch
 import cfnlint.helpers
@@ -99,7 +100,7 @@ def _check_resource(self, cfn, resource_name, resource_values):
             self.logger.debug('Check resource types by region...')
             for region, specs in cfnlint.helpers.RESOURCE_SPECS.items():
                 if region in cfn.regions:
-                    if resource_type not in specs['ResourceTypes']:
+                    if resource_type not in specs['ResourceTypes'] and resource_type not in [s['typeName'] for s in REGISTRY_SCHEMAS]:
                         if not resource_type.startswith(('Custom::', 'AWS::Serverless::')) and not resource_type.endswith('::MODULE'):
                             message = 'Invalid or unsupported Type {0} for resource {1} in {2}'
                             matches.append(RuleMatch(

src/cfnlint/rules/resources/ResourceSchema.py

@@ -0,0 +1,30 @@
+"""
+Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+SPDX-License-Identifier: MIT-0
+"""
+import re
+from jsonschema import validate, ValidationError
+from cfnlint.helpers import REGEX_DYN_REF, PSEUDOPARAMS, FN_PREFIX, UNCONVERTED_SUFFIXES, REGISTRY_SCHEMAS
+from cfnlint.rules import CloudFormationLintRule
+from cfnlint.rules import RuleMatch
+
+class ResourceSchema(CloudFormationLintRule):
+    id = 'E3000'
+    shortdesc = 'Resource schema'
+    description = 'CloudFormation Registry resource schema validation'
+    source_url = 'https://github.com/aws-cloudformation/aws-cloudformation-resource-schema/'
+    tags = ['resources']
+
+    def match(self, cfn):
+        matches = []
+        for schema in REGISTRY_SCHEMAS:
+            resource_type = schema['typeName']
+            for resource_name, resource_values in cfn.get_resources([resource_type]).items():
+                properties = resource_values.get('Properties', {})
+                # ignoring resources with CloudFormation template syntax in Properties
+                if not re.match(REGEX_DYN_REF, str(properties)) and not any(x in str(properties) for x in PSEUDOPARAMS + UNCONVERTED_SUFFIXES) and FN_PREFIX not in str(properties):
+                    try:
+                        validate(properties, schema)
+                    except ValidationError as e:
+                        matches.append(RuleMatch(['Resources', resource_name, 'Properties'], e.message))
+        return matches