diff --git a/src/rpdk/core/generate.py b/src/rpdk/core/generate.py index 55899d04..3afd1154 100644 --- a/src/rpdk/core/generate.py +++ b/src/rpdk/core/generate.py @@ -13,6 +13,7 @@ def generate(_args): project = Project() project.load() project.generate() + project.generate_docs() LOG.warning("Generated files for %s", project.type_name) diff --git a/src/rpdk/core/init.py b/src/rpdk/core/init.py index 1d0c1b88..daf5f29c 100644 --- a/src/rpdk/core/init.py +++ b/src/rpdk/core/init.py @@ -147,6 +147,7 @@ def init(args): project.init(type_name, language) project.generate() + project.generate_docs() LOG.warning("Initialized a new project in %s", project.root.resolve()) diff --git a/src/rpdk/core/project.py b/src/rpdk/core/project.py index 342ebb61..4e511ee0 100644 --- a/src/rpdk/core/project.py +++ b/src/rpdk/core/project.py @@ -1,5 +1,6 @@ import json import logging +import shutil import zipfile from pathlib import Path from tempfile import TemporaryFile @@ -7,7 +8,7 @@ from botocore.exceptions import ClientError, WaiterError from jinja2 import Environment, PackageLoader, select_autoescape -from jsonschema import Draft6Validator +from jsonschema import Draft6Validator, RefResolver from jsonschema.exceptions import ValidationError from .boto_helpers import create_sdk_session @@ -18,6 +19,8 @@ InvalidProjectError, SpecValidationError, ) +from .jsonutils.pointer import fragment_decode, fragment_encode +from .jsonutils.utils import traverse from .plugin_registry import load_plugin from .upload import Uploader @@ -41,12 +44,10 @@ "java8", "java11", "go1.x", - # python2.7 is EOL soon (2020-01-01) "python3.6", "python3.7", "python3.8", "dotnetcore2.1", - # nodejs8.10 is EOL soon (2019-12-31) "nodejs10.x", "nodejs12.x", } @@ -67,6 +68,14 @@ ) +BASIC_TYPE_MAPPINGS = { + "string": "String", + "number": "Double", + "integer": "Double", + "boolean": "Boolean", +} + + class Project: # pylint: disable=too-many-instance-attributes def __init__(self, overwrite_enabled=False, root=None): self.overwrite_enabled = overwrite_enabled @@ -86,7 +95,7 @@ def __init__(self, overwrite_enabled=False, root=None): lstrip_blocks=True, keep_trailing_newline=True, loader=PackageLoader(__name__, "templates/"), - autoescape=select_autoescape(["html", "htm", "xml"]), + autoescape=select_autoescape(["html", "htm", "xml", "md"]), ) LOG.debug("Root directory: %s", self.root) @@ -320,6 +329,128 @@ def submit( f, endpoint_url, region_name, role_arn, use_role, set_default ) + def generate_docs(self): + # generate the docs folder that contains documentation based on the schema + docs_path = self.root / "docs" + + if not self.type_info or not self.schema or "properties" not in self.schema: + LOG.warning( + "Could not generate schema docs due to missing type info or schema" + ) + return + + LOG.debug("Removing generated docs: %s", docs_path) + shutil.rmtree(docs_path, ignore_errors=True) + docs_path.mkdir(exist_ok=True) + + LOG.debug("Writing generated docs") + + # take care not to modify the master schema + docs_schema = json.loads(json.dumps(self.schema)) + + docs_schema["properties"] = { + name: self._set_docs_properties(name, value, (name,)) + for name, value in docs_schema["properties"].items() + } + + LOG.debug("Finished documenting nested properties") + + ref = self._get_docs_primary_identifier(docs_schema) + getatt = self._get_docs_gettable_atts(docs_schema) + + readme_path = docs_path / "README.md" + LOG.debug("Writing docs README: %s", readme_path) + template = self.env.get_template("docs-readme.md") + contents = template.render( + type_name=self.type_name, schema=docs_schema, ref=ref, getatt=getatt + ) + self.safewrite(readme_path, contents) + + @staticmethod + def _get_docs_primary_identifier(docs_schema): + try: + primary_id = docs_schema["primaryIdentifier"] + if len(primary_id) == 1: + # drop /properties + primary_id_path = fragment_decode(primary_id[0], prefix="")[1:] + # at some point, someone might use a nested primary ID + if len(primary_id_path) == 1: + return primary_id_path[0] + LOG.warning("Nested primaryIdentifier found") + except (KeyError, ValueError): + pass + return None + + @staticmethod + def _get_docs_gettable_atts(docs_schema): + def _get_property_description(prop): + path = fragment_decode(prop, prefix="") + name = path[-1] + try: + desc, _resolved_path, _parent = traverse( + docs_schema, path + ("description",) + ) + except (KeyError, IndexError, ValueError): + desc = f"Returns the {name} value." + return {"name": name, "description": desc} + + return [ + _get_property_description(prop) + for prop in docs_schema.get("readOnlyProperties", []) + ] + + def _set_docs_properties(self, propname, prop, proppath): + if "$ref" in prop: + prop = RefResolver.from_schema(self.schema).resolve(prop["$ref"])[1] + + proppath_ptr = fragment_encode(("properties",) + proppath, prefix="") + if ( + "createOnlyProperties" in self.schema + and proppath_ptr in self.schema["createOnlyProperties"] + ): + prop["createonly"] = True + + if prop["type"] in BASIC_TYPE_MAPPINGS: + mapped = BASIC_TYPE_MAPPINGS[prop["type"]] + prop["jsontype"] = prop["yamltype"] = prop["longformtype"] = mapped + elif prop["type"] == "array": + prop["arrayitems"] = arrayitems = self._set_docs_properties( + propname, prop["items"], proppath + ) + prop["jsontype"] = f'[ {arrayitems["jsontype"]}, ... ]' + prop["yamltype"] = f'\n - {arrayitems["longformtype"]}' + prop["longformtype"] = f'List of {arrayitems["longformtype"]}' + elif prop["type"] == "object": + template = self.env.get_template("docs-subproperty.md") + docs_path = self.root / "docs" + + prop["properties"] = { + name: self._set_docs_properties(name, value, proppath + (name,)) + for name, value in prop["properties"].items() + } + + subproperty_name = " ".join(proppath) + subproperty_filename = "-".join(proppath).lower() + ".md" + subproperty_path = docs_path / subproperty_filename + + LOG.debug("Writing docs %s: %s", subproperty_filename, subproperty_path) + contents = template.render( + type_name=self.type_name, subproperty_name=subproperty_name, schema=prop + ) + self.safewrite(subproperty_path, contents) + + href = f'{propname}' + prop["jsontype"] = prop["yamltype"] = prop["longformtype"] = href + else: + prop["jsontype"] = "Unknown" + prop["yamltype"] = "Unknown" + prop["longformtype"] = "Unknown" + + if "enum" in prop: + prop["allowedvalues"] = prop["enum"] + + return prop + def _upload( self, fileobj, endpoint_url, region_name, role_arn, use_role, set_default ): # pylint: disable=too-many-arguments, too-many-locals diff --git a/src/rpdk/core/templates/docs-readme.md b/src/rpdk/core/templates/docs-readme.md new file mode 100644 index 00000000..3a89d2b7 --- /dev/null +++ b/src/rpdk/core/templates/docs-readme.md @@ -0,0 +1,105 @@ +# {{ type_name }} +{% if schema.description %} + +{{ schema.description }} +{% endif %} + +## Syntax + +To declare this entity in your AWS CloudFormation template, use the following syntax: + +### JSON + +
+{
+    "Type" : "{{ type_name }}",
+    "Properties" : {
+{% if schema.properties %}
+{% for propname, prop in schema.properties.items() %}
+        "{{ propname }}" : {{ prop.jsontype }}{% if not loop.last %},{% endif %}
+
+{% endfor %}
+{% endif %}
+    }
+}
+
+ +### YAML + +
+Type: {{ type_name }}
+Properties:
+{% if schema.properties %}
+{% for propname, prop in schema.properties.items() %}
+    {{ propname }}: {{ prop.yamltype }}
+{% endfor %}
+{% endif %}
+
+{% if schema.properties %} + +## Properties + +{% for propname, prop in schema.properties.items() %} +#### {{ propname }} +{% if prop.description %} + +{{ prop.description }} +{% endif %} + +_Required_: {% if propname in schema.required %}Yes{% else %}No{% endif %} + + +_Type_: {{ prop.longformtype }} +{% if prop.allowedvalues %} + +_Allowed Values_: {% for allowedvalue in prop.allowedvalues %}{{ allowedvalue }}{% if not loop.last %} | {% endif %}{% endfor %} + +{% endif %} +{% if prop.minLength %} + +_Minimum_: {{ prop.minLength }} +{% endif %} +{% if prop.maxLength %} + +_Maximum_: {{ prop.maxLength }} +{% endif %} +{% if prop.pattern %} + +_Pattern_: {{ prop.pattern }} +{% endif %} +{% if prop.createonly %} + +_Update requires_: [Replacement](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-replacement) +{% else %} + +_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) +{% endif %} + +{% endfor %} +{% endif %} +{% if getatt or ref %} +## Return Values +{% if ref %} + +### Ref + +When you pass the logical ID of this resource to the intrinsic `Ref` function, Ref returns the {{ ref }}. +{% endif %} +{% if getatt %} + +### Fn::GetAtt + +The `Fn::GetAtt` intrinsic function returns a value for a specified attribute of this type. The following are the available attributes and sample return values. + +For more information about using the `Fn::GetAtt` intrinsic function, see [Fn::GetAtt](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/intrinsic-function-reference-getatt.html). + +{% for att in getatt %} +#### {{ att.name }} +{% if att.description %} + +{{ att.description }} +{% endif %} + +{% endfor %} +{% endif %} +{% endif %} diff --git a/src/rpdk/core/templates/docs-subproperty.md b/src/rpdk/core/templates/docs-subproperty.md new file mode 100644 index 00000000..344403f8 --- /dev/null +++ b/src/rpdk/core/templates/docs-subproperty.md @@ -0,0 +1,73 @@ +# {{ type_name }} {{ subproperty_name }} +{% if schema.description %} + +{{ schema.description }} +{% endif %} + +## Syntax + +To declare this entity in your AWS CloudFormation template, use the following syntax: + +### JSON + +
+{
+{% if schema.properties %}
+{% for propname, prop in schema.properties.items() %}
+    "{{ propname }}" : {{ prop.jsontype }}{% if not loop.last %},{% endif %}
+
+{% endfor %}
+{% endif %}
+}
+
+ +### YAML + +
+{% if schema.properties %}
+{% for propname, prop in schema.properties.items() %}
+{{ propname }}: {{ prop.yamltype }}
+{% endfor %}
+{% endif %}
+
+{% if schema.properties %} + +## Properties + +{% for propname, prop in schema.properties.items() %} +#### {{ propname }} +{% if prop.description %} + +{{ prop.description }} +{% endif %} + +_Required_: {% if propname in schema.required %}Yes{% else %}No{% endif %} + +_Type_: {{ prop.longformtype }} +{% if prop.allowedvalues %} + +_Allowed Values_: {% for allowedvalue in prop.allowedvalues %}{{ allowedvalue }}{% if not loop.last %} | {% endif %}{% endfor %} + +{% endif %} +{% if prop.minLength %} + +_Minimum_: {{ prop.minLength }} +{% endif %} +{% if prop.maxLength %} + +_Maximum_: {{ prop.maxLength }} +{% endif %} +{% if prop.pattern %} + +_Pattern_: {{ prop.pattern }} +{% endif %} +{% if prop.createonly %} + +_Update requires_: [Replacement](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-replacement) +{% else %} + +_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) +{% endif %} + +{% endfor %} +{% endif %} diff --git a/tests/data/schema/invalid/invalid_definition_reference_invalid.json b/tests/data/schema/invalid/invalid_definition_reference_invalid.json new file mode 100644 index 00000000..4fcdc162 --- /dev/null +++ b/tests/data/schema/invalid/invalid_definition_reference_invalid.json @@ -0,0 +1,14 @@ +{ + "typeName": "AWS::Valid::TypeName", + "description": "a test schema", + "properties": { + "property1": { + "type": "string", + "$ref": "#/definitions/invalid" + } + }, + "primaryIdentifier": [ + "/properties/property1" + ], + "additionalProperties": false +} diff --git a/tests/data/schema/invalid/invalid_property_type_invalid.json b/tests/data/schema/invalid/invalid_property_type_invalid.json new file mode 100644 index 00000000..88e9c513 --- /dev/null +++ b/tests/data/schema/invalid/invalid_property_type_invalid.json @@ -0,0 +1,13 @@ +{ + "typeName": "AWS::Valid::TypeName", + "description": "a test schema", + "properties": { + "property1": { + "type": "invalid" + } + }, + "primaryIdentifier": [ + "/properties/property1" + ], + "additionalProperties": false +} diff --git a/tests/data/schema/valid/valid_type_complex.json b/tests/data/schema/valid/valid_type_complex.json new file mode 100644 index 00000000..bd31c172 --- /dev/null +++ b/tests/data/schema/valid/valid_type_complex.json @@ -0,0 +1,64 @@ +{ + "typeName": "AWS::Valid::TypeName", + "description": "a test schema", + "definitions": { + "obj2def": { + "type": "object", + "additionalProperties": false, + "properties": { + "str1": { + "type": "string", + "minLength": 2 + } + } + } + }, + "properties": { + "enum1": { + "type": "string", + "enum": [ + "test", + "test2" + ] + }, + "str2": { + "type": "string", + "description": "some description" + }, + "obj1": { + "type": "object", + "description": "", + "$ref": "#/definitions/obj2def" + }, + "str3": { + "type": "string", + "description": "" + }, + "arr1": { + "type": "array", + "description": "", + "items": { + "$ref": "#/definitions/obj2def" + } + } + }, + "primaryIdentifier": [ + "/properties/enum1" + ], + "additionalIdentifiers": [ + [ + "/properties/enum1", + "/properties/str2" + ], + [ + "/properties/obj1/obj2/str1" + ], + [ + "/properties/str2" + ] + ], + "createOnlyProperties": [ + "/properties/str2" + ], + "additionalProperties": false +} diff --git a/tests/data/schema/valid/valid_type_composite_primary_identifier.json b/tests/data/schema/valid/valid_type_composite_primary_identifier.json new file mode 100644 index 00000000..beb4f49a --- /dev/null +++ b/tests/data/schema/valid/valid_type_composite_primary_identifier.json @@ -0,0 +1,17 @@ +{ + "typeName": "AWS::Valid::TypeName", + "description": "a test schema", + "properties": { + "property1": { + "type": "string" + }, + "property2": { + "type": "string" + } + }, + "primaryIdentifier": [ + "/properties/property1", + "/properties/property2" + ], + "additionalProperties": false +} diff --git a/tests/test_generate.py b/tests/test_generate.py index a7ac94a3..f10ad1f7 100644 --- a/tests/test_generate.py +++ b/tests/test_generate.py @@ -13,6 +13,7 @@ def test_generate_command_generate(capsys): mock_project.load.assert_called_once_with() mock_project.generate.assert_called_once_with() + mock_project.generate_docs.assert_called_once_with() out, err = capsys.readouterr() assert not err diff --git a/tests/test_project.py b/tests/test_project.py index 6ec37742..726f720c 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -14,6 +14,7 @@ import yaml from botocore.exceptions import ClientError, WaiterError +from rpdk.core.data_loaders import resource_json from rpdk.core.exceptions import ( DownstreamError, InternalError, @@ -172,9 +173,154 @@ def test_generate_no_handlers(project): mock_plugin = MagicMock(spec=["generate"]) with patch.object(project, "_plugin", mock_plugin): project.generate() + project.generate_docs() mock_plugin.generate.assert_called_once_with(project) +def test_generate_with_docs(project, tmp_path_factory): + project.schema = resource_json( + __name__, "data/schema/valid/valid_type_complex.json" + ) + project.type_name = "AWS::Color::Red" + # tmpdir conflicts with other tests, make a unique one + project.root = tmp_path_factory.mktemp("generate_with_docs") + mock_plugin = MagicMock(spec=["generate"]) + with patch.object(project, "_plugin", mock_plugin): + project.generate() + project.generate_docs() + mock_plugin.generate.assert_called_once_with(project) + + docs_dir = project.root / "docs" + readme_file = project.root / "docs" / "README.md" + + assert docs_dir.is_dir() + assert readme_file.is_file() + with patch.object(project, "_plugin", mock_plugin): + project.generate() + project.generate_docs() + readme_contents = readme_file.read_text(encoding="utf-8") + assert project.type_name in readme_contents + + +def test_generate_with_docs_nested_object(project, tmp_path_factory): + project.schema = resource_json( + __name__, "data/schema/valid/valid_nested_property_object.json" + ) + project.type_name = "AWS::Color::Red" + # tmpdir conflicts with other tests, make a unique one + project.root = tmp_path_factory.mktemp("generate_with_docs_nested_object") + mock_plugin = MagicMock(spec=["generate"]) + with patch.object(project, "_plugin", mock_plugin): + project.generate() + project.generate_docs() + mock_plugin.generate.assert_called_once_with(project) + + docs_dir = project.root / "docs" + readme_file = project.root / "docs" / "README.md" + + assert docs_dir.is_dir() + assert readme_file.is_file() + with patch.object(project, "_plugin", mock_plugin): + project.generate() + project.generate_docs() + readme_contents = readme_file.read_text(encoding="utf-8") + assert project.type_name in readme_contents + + +def test_generate_with_docs_invalid_property_type(project, tmp_path_factory): + project.schema = resource_json( + __name__, "data/schema/invalid/invalid_property_type_invalid.json" + ) + project.type_name = "AWS::Color::Red" + # tmpdir conflicts with other tests, make a unique one + project.root = tmp_path_factory.mktemp("generate_with_docs_invalid_property_type") + mock_plugin = MagicMock(spec=["generate"]) + with patch.object(project, "_plugin", mock_plugin): + # skip actual generation + project.generate_docs() + + docs_dir = project.root / "docs" + readme_file = project.root / "docs" / "README.md" + + assert docs_dir.is_dir() + assert readme_file.is_file() + with patch.object(project, "_plugin", mock_plugin): + project.generate() + project.generate_docs() + readme_contents = readme_file.read_text(encoding="utf-8") + assert project.type_name in readme_contents + + +def test_generate_with_docs_composite_primary_identifier(project, tmp_path_factory): + project.schema = resource_json( + __name__, "data/schema/valid/valid_type_composite_primary_identifier.json" + ) + project.type_name = "AWS::Color::Red" + # tmpdir conflicts with other tests, make a unique one + project.root = tmp_path_factory.mktemp( + "generate_with_docs_composite_primary_identifier" + ) + mock_plugin = MagicMock(spec=["generate"]) + with patch.object(project, "_plugin", mock_plugin): + project.generate() + project.generate_docs() + mock_plugin.generate.assert_called_once_with(project) + + docs_dir = project.root / "docs" + readme_file = project.root / "docs" / "README.md" + + assert docs_dir.is_dir() + assert readme_file.is_file() + with patch.object(project, "_plugin", mock_plugin): + project.generate() + project.generate_docs() + readme_contents = readme_file.read_text(encoding="utf-8") + assert project.type_name in readme_contents + + +def test_generate_with_docs_no_type(project, tmp_path_factory): + project.schema = {"properties": {}} + # tmpdir conflicts with other tests, make a unique one + project.root = tmp_path_factory.mktemp("generate_with_docs_no_type") + mock_plugin = MagicMock(spec=["generate"]) + with patch.object(project, "_plugin", mock_plugin): + project.generate() + project.generate_docs() + mock_plugin.generate.assert_called_once_with(project) + + docs_dir = project.root / "docs" + + assert not docs_dir.is_dir() + + +def test_generate_with_docs_twice(project, tmp_path_factory): + project.schema = {"properties": {}} + project.type_name = "AWS::Color::Red" + # tmpdir conflicts with other tests, make a unique one + project.root = tmp_path_factory.mktemp("generate_with_docs_twice") + mock_plugin = MagicMock(spec=["generate"]) + with patch.object(project, "_plugin", mock_plugin): + project.generate() + project.generate_docs() + mock_plugin.generate.assert_called_once_with(project) + + docs_dir = project.root / "docs" + readme_file = docs_dir / "README.md" + + assert docs_dir.is_dir() + assert readme_file.is_file() + with patch.object(project, "_plugin", mock_plugin): + project.generate() + project.generate_docs() + assert docs_dir.is_dir() + assert readme_file.is_file() + with patch.object(project, "_plugin", mock_plugin): + project.generate() + project.generate_docs() + readme_contents = readme_file.read_text(encoding="utf-8") + assert project.type_name in readme_contents + + def test_generate_handlers(project, tmpdir): project.type_name = "Test::Handler::Test" expected_actions = {"createAction", "readAction"} @@ -729,3 +875,53 @@ def test__write_settings_invalid_runtime(project): with pytest.raises(InternalError): project.write_settings() + + +@pytest.mark.parametrize( + "docs_schema", + ( + {}, + {"primaryIdentifier": ["/properties/Id1", "/properties/Id1"]}, + {"primaryIdentifier": ["/properties/Nested/Id1"]}, + ), +) +def test__get_docs_primary_identifier_bad_path(docs_schema): + ref = Project._get_docs_primary_identifier(docs_schema) + assert ref is None + + +def test__get_docs_primary_identifier_good_path(): + ref = Project._get_docs_primary_identifier( + {"primaryIdentifier": ["/properties/Id1"]} + ) + assert ref == "Id1" + + +def test__get_docs_gettable_atts_empty(): + getatt = Project._get_docs_gettable_atts({}) + assert getatt == [] + + +@pytest.mark.parametrize( + "docs_schema", + ( + {"readOnlyProperties": ["/properties/Id2"]}, + {"properties": {}, "readOnlyProperties": ["/properties/Id2"]}, + {"properties": {"Id2": {}}, "readOnlyProperties": ["/properties/Id2"]}, + ), +) +def test__get_docs_gettable_atts_bad_path(docs_schema): + getatt = Project._get_docs_gettable_atts(docs_schema) + assert getatt == [ + {"name": "Id2", "description": "Returns the Id2 value."} + ] + + +def test__get_docs_gettable_atts_good_path(): + getatt = Project._get_docs_gettable_atts( + { + "properties": {"Id2": {"description": "foo"}}, + "readOnlyProperties": ["/properties/Id2"], + } + ) + assert getatt == [{"name": "Id2", "description": "foo"}]