From db6e85813ed0925b6e259845fcf864f6c8485f3c Mon Sep 17 00:00:00 2001 From: Andrew Shamah <42912128+amshamah419@users.noreply.github.com> Date: Thu, 30 Apr 2020 22:03:40 +0300 Subject: [PATCH] Add Update Release Notes command (#338) * Add *.md files from ReleaseNotes to Artifacts * Add *.md files from ReleaseNotes to Artifacts * Linting * Fix imports * Remove unnecessary prints * Ran isort * Revert: Ran isort (f4d53e14) * Revert: Revert: Ran isort (f4d53e14) (ace6c7b2) * Import mess * Import mess * ugh * ugh * ughhhh * Move constants to common constants * begin add tests * begin add tests * checking tests * checking tests * trying to squeak over the test coverage threshold * eof * cover this * Changes per Design Review * Changes per Design Review * Fix tests * Testing gpg * Testing gpg * Remove unnecessary line * Updates per CR * Improve unit testing, add ability to iterate over list of packs * Preserve pack_metadata while testing * Add check for errors, update init template * flake * Fix mocker import * Coverage... * Please coverall gods, don't fail this build needlessly * Changes per CR * Changes per CR * Changes per CR * Fix tests * Fixed bug * Changes per CR * Add more unit tests * Add even more unit tests --- .coveragerc | 1 + demisto_sdk/__main__.py | 96 ++++- demisto_sdk/commands/common/constants.py | 12 + .../create_artifacts/content_creator.py | 27 ++ demisto_sdk/commands/init/initiator.py | 6 +- .../commands/init/tests/initiator_test.py | 6 +- .../commands/update_release_notes/__init__.py | 0 .../tests/update_rn_test.py | 351 ++++++++++++++++++ .../update_release_notes/update_rn.py | 212 +++++++++++ .../commands/validate/file_validator.py | 1 - .../test_files/Integration/HelloWorld.py | 0 .../test_files/Integration/HelloWorld.yml | 0 .../tests/test_files/ReleaseNotes/1_1_1.md | 1 + .../test_files/fake_pack/pack_metadata.json | 9 +- .../test_files/fake_pack_invalid/.pack-ignore | 0 .../fake_pack_invalid/.secrets-ignore | 1 + .../Integrations/__init__.py | 0 .../Integrations/fake_changelog/__init__.py | 0 .../test_files/fake_pack_invalid/README.md | 2 + .../test_files/fake_pack_invalid/__init__.py | 0 .../fake_pack_invalid/pack_metadata.json | 33 ++ 21 files changed, 747 insertions(+), 11 deletions(-) create mode 100644 demisto_sdk/commands/update_release_notes/__init__.py create mode 100644 demisto_sdk/commands/update_release_notes/tests/update_rn_test.py create mode 100644 demisto_sdk/commands/update_release_notes/update_rn.py create mode 100644 demisto_sdk/tests/test_files/Integration/HelloWorld.py create mode 100644 demisto_sdk/tests/test_files/Integration/HelloWorld.yml create mode 100644 demisto_sdk/tests/test_files/ReleaseNotes/1_1_1.md create mode 100644 demisto_sdk/tests/test_files/fake_pack_invalid/.pack-ignore create mode 100644 demisto_sdk/tests/test_files/fake_pack_invalid/.secrets-ignore create mode 100644 demisto_sdk/tests/test_files/fake_pack_invalid/Integrations/__init__.py create mode 100644 demisto_sdk/tests/test_files/fake_pack_invalid/Integrations/fake_changelog/__init__.py create mode 100644 demisto_sdk/tests/test_files/fake_pack_invalid/README.md create mode 100644 demisto_sdk/tests/test_files/fake_pack_invalid/__init__.py create mode 100644 demisto_sdk/tests/test_files/fake_pack_invalid/pack_metadata.json diff --git a/.coveragerc b/.coveragerc index 4b03ca75fba..a9c4220c84c 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,6 +2,7 @@ omit = **/tests/** **/__init__.py + demisto_sdk/__main__.py [html] directory = coverage_html_report diff --git a/demisto_sdk/__main__.py b/demisto_sdk/__main__.py index 406e1a2b7cf..6a5b35e5ba2 100644 --- a/demisto_sdk/__main__.py +++ b/demisto_sdk/__main__.py @@ -1,5 +1,6 @@ # Site packages import os +import re import sys from pkg_resources import get_distribution @@ -10,7 +11,9 @@ # Common tools from demisto_sdk.commands.common.tools import (find_type, get_last_remote_release_version, - print_error, print_warning) + get_pack_name, + pack_name_to_path, print_error, + print_warning) from demisto_sdk.commands.create_artifacts.content_creator import \ ContentCreator from demisto_sdk.commands.create_id_set.create_id_set import IDSetCreator @@ -36,6 +39,7 @@ from demisto_sdk.commands.secrets.secrets import SecretsValidator from demisto_sdk.commands.split_yml.extractor import Extractor from demisto_sdk.commands.unify.unifier import Unifier +from demisto_sdk.commands.update_release_notes.update_rn import UpdateRN from demisto_sdk.commands.upload.uploader import Uploader from demisto_sdk.commands.validate.file_validator import FilesValidator @@ -49,6 +53,28 @@ def __init__(self): self.configuration = None +class RNInputValidation(click.ParamType): + name = 'update_type' + + def validate_rn_input(self, value, param, ctx): + if value: + if re.match(r'(?i)(?<=|^)major(?= |$)', value): + update_type = 'major' + elif re.match(r'(?i)(?<=|^)minor(?= |$)', value): + update_type = 'minor' + elif re.match(r'(?i)(?<=|^)revision(?= |$)', value): + update_type = 'revision' + else: + self.fail( + f'{value} is not a valid option. Please select: major, minor, revision', + param, + ctx, + ) + else: + update_type = 'revision' + return update_type + + pass_config = click.make_pass_decorator(DemistoSDK, ensure=True) @@ -684,6 +710,74 @@ def id_set_command(**kwargs): id_set_creator.create_id_set() +# ====================== update-release-notes =================== # +@main.command(name="update-release-notes", + short_help='''Auto-increment pack version and generate release notes template.''') +@click.help_option( + '-h', '--help' +) +@click.option( + "-p", "--pack", help="Name of the pack." +) +@click.option( + '-u', '--update_type', help="The type of update being done. [major, minor, revision]", + type=RNInputValidation() +) +@click.option( + '--all', help="Update all changed packs", is_flag=True +) +@click.option( + "--pre_release", help="Indicates that this change should be designated a pre-release version.", + is_flag=True) +def update_pack_releasenotes(**kwargs): + _pack = kwargs.get('pack') + update_type = kwargs.get('update_type') + pre_release = kwargs.get('pre_release') + is_all = kwargs.get('all') + modified, added, old, _packs = FilesValidator(use_git=True).get_modified_and_added_files() + packs_existing_rn = set() + for pf in added: + if 'ReleaseNotes' in pf: + pack_with_existing_rn = get_pack_name(pf) + packs_existing_rn.add(pack_with_existing_rn) + if len(packs_existing_rn): + existing_rns = ''.join(f"{p}, " for p in packs_existing_rn) + print_warning(f"Found existing release notes for the following packs: {existing_rns.rstrip(', ')}") + if len(_packs) > 1: + pack_list = ''.join(f"{p}, " for p in _packs) + if not is_all: + if _pack: + pass + else: + print_error(f"Detected changes in the following packs: {pack_list.rstrip(', ')}\n" + f"To update release notes in a specific pack, please use the -p parameter " + f"along with the pack name.") + sys.exit(0) + if len(modified) < 1: + print_warning('No changes were detected.') + sys.exit(0) + if is_all and not _pack: + packs = list(_packs - packs_existing_rn) + packs_list = ''.join(f"{p}, " for p in packs) + print_warning(f"Adding release notes to the following packs: {packs_list.rstrip(', ')}") + for pack in packs: + update_pack_rn = UpdateRN(pack=pack, update_type=update_type, pack_files=modified, + pre_release=pre_release) + update_pack_rn.execute_update() + elif is_all and _pack: + print_error(f"Please remove the --all flag when specifying only one pack.") + sys.exit(0) + else: + if _pack: + if _pack in packs_existing_rn: + print_error(f"New release notes file already found for {_pack}. " + f"Please update manually or delete {pack_name_to_path(_pack)}") + else: + update_pack_rn = UpdateRN(pack=_pack, update_type=update_type, pack_files=modified, + pre_release=pre_release) + update_pack_rn.execute_update() + + # ====================== find-dependencies ====================== # @main.command(name="find-dependencies", short_help='''Find pack dependencies and update pack metadata.''') diff --git a/demisto_sdk/commands/common/constants.py b/demisto_sdk/commands/common/constants.py index d508945064a..8cf294f2466 100644 --- a/demisto_sdk/commands/common/constants.py +++ b/demisto_sdk/commands/common/constants.py @@ -269,6 +269,7 @@ def found_hidden_param(parameter_name): BETA_INTEGRATIONS_DIR = 'Beta_Integrations' PACKS_DIR = 'Packs' TOOLS_DIR = 'Tools' +RELEASE_NOTES_DIR = 'ReleaseNotes' TESTS_DIR = 'Tests' SCRIPT = 'script' @@ -555,6 +556,7 @@ def found_hidden_param(parameter_name): PACKS_WIDGETS_REGEX = r'{}{}/([^/]+)/{}/([^.]+)\.json'.format(CAN_START_WITH_DOT_SLASH, PACKS_DIR, WIDGETS_DIR) PACKS_REPORTS_REGEX = r'{}/([^/]+)/{}/([^.]+)\.json'.format(PACKS_DIR, REPORTS_DIR) PACKS_CHANGELOG_REGEX = r'{}{}/([^/]+)/CHANGELOG\.md$'.format(CAN_START_WITH_DOT_SLASH, PACKS_DIR) +PACKS_RELEASE_NOTES_REGEX = r'{}{}/([^/]+)/{}/([^/]+)\.md$'.format(CAN_START_WITH_DOT_SLASH, PACKS_DIR, RELEASE_NOTES_DIR) PACKS_README_REGEX = r'{}{}/([^/]+)/README\.md'.format(CAN_START_WITH_DOT_SLASH, PACKS_DIR) PACKS_README_REGEX_INNER = r'{}{}/([^/]+)/([^/]+)/([^/]+)/README\.md'.format(CAN_START_WITH_DOT_SLASH, PACKS_DIR) @@ -846,6 +848,16 @@ def found_hidden_param(parameter_name): PACKS_README_REGEX, PACKS_README_REGEX_INNER, INTEGRATION_OLD_README_REGEX, + # Pack Misc + PACKS_CLASSIFIERS_REGEX, + PACKS_DASHBOARDS_REGEX, + PACKS_INCIDENT_TYPES_REGEX, + PACKS_INCIDENT_FIELDS_REGEX, + PACKS_INDICATOR_FIELDS_REGEX, + PACKS_LAYOUTS_REGEX, + PACKS_WIDGETS_REGEX, + PACKS_REPORTS_REGEX, + PACKS_RELEASE_NOTES_REGEX ] CHECKED_TYPES_NO_REGEX = [item.replace(CAN_START_WITH_DOT_SLASH, "").replace(NOT_TEST, "") for item in diff --git a/demisto_sdk/commands/create_artifacts/content_creator.py b/demisto_sdk/commands/create_artifacts/content_creator.py index 38fe5cbe7f2..966c6e768ab 100644 --- a/demisto_sdk/commands/create_artifacts/content_creator.py +++ b/demisto_sdk/commands/create_artifacts/content_creator.py @@ -19,6 +19,7 @@ INTEGRATIONS_DIR, LAYOUTS_DIR, MISC_DIR, PACKS_DIR, PLAYBOOKS_DIR, + RELEASE_NOTES_DIR, REPORTS_DIR, SCRIPTS_DIR, TEST_PLAYBOOKS_DIR, TOOLS_DIR, WIDGETS_DIR) @@ -244,6 +245,30 @@ def copy_dir_json(self, dir_path, bundle): shutil.copyfile(path, os.path.join(bundle, dpath)) + def copy_dir_md(self, dir_path, bundle): + """ + Copy the md files inside a directory to a bundle. + + :param dir_path: source directory + :param bundle: destination bundle + :return: None + """ + # handle *.md files + dir_name = os.path.basename(dir_path) + scan_files = glob.glob(os.path.join(dir_path, '*.md')) + for path in scan_files: + new_path = os.path.basename(path) + if dir_name == RELEASE_NOTES_DIR: + if os.path.isfile(os.path.join(bundle, new_path)): + raise NameError( + f'Failed while trying to create {os.path.join(bundle, new_path)}. File already exists.' + ) + + if len(new_path) >= self.file_name_max_size: + self.long_file_names.append(os.path.basename(new_path)) + + shutil.copyfile(path, os.path.join(bundle, new_path)) + def copy_dir_files(self, *args): """ Copy the yml and json files from inside a directory to a bundle. @@ -255,6 +280,8 @@ def copy_dir_files(self, *args): self.copy_dir_json(*args) # handle *.yml files self.copy_dir_yml(*args) + # handle *.md files + self.copy_dir_md(*args) def copy_test_files(self, test_playbooks_dir=TEST_PLAYBOOKS_DIR): """ diff --git a/demisto_sdk/commands/init/initiator.py b/demisto_sdk/commands/init/initiator.py index 92ba11d777d..5ff43ba5bda 100644 --- a/demisto_sdk/commands/init/initiator.py +++ b/demisto_sdk/commands/init/initiator.py @@ -192,10 +192,10 @@ def create_metadata(fill_manually: bool) -> Dict: metadata = { 'name': '## FILL OUT MANUALLY ##', 'description': '## FILL OUT MANUALLY ##', - 'support': 'demisto', + 'support': 'xsoar', 'currentVersion': PACK_INITIAL_VERSION, - 'author': 'demisto', - 'url': 'https://www.demisto.com', + 'author': 'Cortex XSOAR', + 'url': 'https://www.paloaltonetworks.com/cortex', 'email': '', 'categories': [], 'tags': [], diff --git a/demisto_sdk/commands/init/tests/initiator_test.py b/demisto_sdk/commands/init/tests/initiator_test.py index 0d6f22599ed..672d48181be 100644 --- a/demisto_sdk/commands/init/tests/initiator_test.py +++ b/demisto_sdk/commands/init/tests/initiator_test.py @@ -71,10 +71,10 @@ def test_create_metadata(monkeypatch, initiator): assert pack_metadata == { 'name': '## FILL OUT MANUALLY ##', 'description': '## FILL OUT MANUALLY ##', - 'support': 'demisto', + 'support': 'xsoar', 'currentVersion': PACK_INITIAL_VERSION, - 'author': 'demisto', - 'url': 'https://www.demisto.com', + 'author': 'Cortex XSOAR', + 'url': 'https://www.paloaltonetworks.com/cortex', 'email': '', 'categories': [], 'tags': [], diff --git a/demisto_sdk/commands/update_release_notes/__init__.py b/demisto_sdk/commands/update_release_notes/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/demisto_sdk/commands/update_release_notes/tests/update_rn_test.py b/demisto_sdk/commands/update_release_notes/tests/update_rn_test.py new file mode 100644 index 00000000000..a9f3c51271f --- /dev/null +++ b/demisto_sdk/commands/update_release_notes/tests/update_rn_test.py @@ -0,0 +1,351 @@ +import os +import shutil +import unittest + +from demisto_sdk.commands.common.git_tools import git_path + + +class TestRNUpdate(unittest.TestCase): + FILES_PATH = os.path.normpath(os.path.join(__file__, f'{git_path()}/demisto_sdk/tests', 'test_files')) + + def test_build_rn_template_integration(self): + """ + Given: + - a dict of changed items + When: + - we want to produce a release notes template + Then: + - return a markdown string + """ + expected_result = "\n#### Integrations\n- __Hello World Integration__\n%%UPDATE_RN%%\n" \ + "\n#### Playbooks\n- __Hello World Playbook__\n%%UPDATE_RN%%\n" \ + "\n#### Scripts\n- __Hello World Script__\n%%UPDATE_RN%%\n" \ + "\n#### IncidentFields\n- __Hello World IncidentField__\n%%UPDATE_RN%%\n" \ + "\n#### Classifiers\n- __Hello World Classifier__\n%%UPDATE_RN%%\n" \ + "\n#### Layouts\n- __Hello World Layout__\n%%UPDATE_RN%%\n" \ + "\n#### IncidentTypes\n- __Hello World Incident Type__\n%%UPDATE_RN%%\n" + from demisto_sdk.commands.update_release_notes.update_rn import UpdateRN + update_rn = UpdateRN(pack="HelloWorld", update_type='minor', pack_files={'HelloWorld'}) + changed_items = { + "Hello World Integration": "Integration", + "Hello World Playbook": "Playbook", + "Hello World Script": "Script", + "Hello World IncidentField": "IncidentFields", + "Hello World Classifier": "Classifiers", + "N/A": "Integration", + "Hello World Layout": "Layouts", + "Hello World Incident Type": "IncidentTypes", + } + release_notes = update_rn.build_rn_template(changed_items) + assert expected_result == release_notes + + def test_find_corresponding_yml(self): + """ + Given: + - a filepath containing a python file + When: + - determining the changed file + Then: + - return only the yml of the changed file + """ + expected_result = "Integration/HelloWorld.yml" + from demisto_sdk.commands.update_release_notes.update_rn import UpdateRN + update_rn = UpdateRN(pack="HelloWorld", update_type='minor', pack_files={'HelloWorld'}) + filepath = 'Integration/HelloWorld.py' + filename = update_rn.find_corresponding_yml(filepath) + assert expected_result == filename + + def test_return_release_notes_path(self): + """ + Given: + - a pack name and version + When: + - building the release notes file within the ReleaseNotes directory + Then: + - the filepath of the correct release notes. + """ + expected_result = 'Packs/HelloWorld/ReleaseNotes/1_1_1.md' + from demisto_sdk.commands.update_release_notes.update_rn import UpdateRN + update_rn = UpdateRN(pack="HelloWorld", update_type='minor', pack_files={'HelloWorld'}) + input_version = '1.1.1' + result = update_rn.return_release_notes_path(input_version) + assert expected_result == result + + def test_bump_version_number_minor(self): + """ + Given: + - a pack name and version + When: + - bumping the version number in the metadata.json + Then: + - return the correct bumped version number + """ + shutil.copy(src=os.path.join(TestRNUpdate.FILES_PATH, 'fake_pack/pack_metadata.json'), + dst=os.path.join(TestRNUpdate.FILES_PATH, 'fake_pack/_pack_metadata.json')) + expected_version = '1.1.0' + from demisto_sdk.commands.update_release_notes.update_rn import UpdateRN + update_rn = UpdateRN(pack="HelloWorld", update_type='minor', pack_files={'HelloWorld'}) + update_rn.metadata_path = os.path.join(TestRNUpdate.FILES_PATH, 'fake_pack/pack_metadata.json') + version_number = update_rn.bump_version_number(pre_release=False) + assert version_number == expected_version + os.remove(os.path.join(TestRNUpdate.FILES_PATH, 'fake_pack/pack_metadata.json')) + shutil.copy(src=os.path.join(TestRNUpdate.FILES_PATH, 'fake_pack/_pack_metadata.json'), + dst=os.path.join(TestRNUpdate.FILES_PATH, 'fake_pack/pack_metadata.json')) + + def test_bump_version_number_major(self): + """ + Given: + - a pack name and version + When: + - bumping the version number in the metadata.json + Then: + - return the correct bumped version number + """ + shutil.copy(src=os.path.join(TestRNUpdate.FILES_PATH, 'fake_pack/pack_metadata.json'), + dst=os.path.join(TestRNUpdate.FILES_PATH, 'fake_pack/_pack_metadata.json')) + expected_version = '2.0.0' + from demisto_sdk.commands.update_release_notes.update_rn import UpdateRN + update_rn = UpdateRN(pack="HelloWorld", update_type='major', pack_files={'HelloWorld'}) + update_rn.metadata_path = os.path.join(TestRNUpdate.FILES_PATH, 'fake_pack/pack_metadata.json') + version_number = update_rn.bump_version_number(pre_release=False) + assert version_number == expected_version + os.remove(os.path.join(TestRNUpdate.FILES_PATH, 'fake_pack/pack_metadata.json')) + shutil.copy(src=os.path.join(TestRNUpdate.FILES_PATH, 'fake_pack/_pack_metadata.json'), + dst=os.path.join(TestRNUpdate.FILES_PATH, 'fake_pack/pack_metadata.json')) + + def test_bump_version_number_revision(self): + """ + Given: + - a pack name and version + When: + - bumping the version number in the metadata.json + Then: + - return the correct bumped version number + """ + shutil.copy(src=os.path.join(TestRNUpdate.FILES_PATH, 'fake_pack/pack_metadata.json'), + dst=os.path.join(TestRNUpdate.FILES_PATH, 'fake_pack/_pack_metadata.json')) + expected_version = '1.0.1' + from demisto_sdk.commands.update_release_notes.update_rn import UpdateRN + update_rn = UpdateRN(pack="HelloWorld", update_type='revision', pack_files={'HelloWorld'}) + update_rn.metadata_path = os.path.join(TestRNUpdate.FILES_PATH, 'fake_pack/pack_metadata.json') + version_number = update_rn.bump_version_number(pre_release=False) + assert version_number == expected_version + os.remove(os.path.join(TestRNUpdate.FILES_PATH, 'fake_pack/pack_metadata.json')) + shutil.copy(src=os.path.join(TestRNUpdate.FILES_PATH, 'fake_pack/_pack_metadata.json'), + dst=os.path.join(TestRNUpdate.FILES_PATH, 'fake_pack/pack_metadata.json')) + + def test_bump_version_number_revision_overflow(self): + """ + Given: + - a pack name and a version before an overflow condition + When: + - bumping the version number in the metadata.json + Then: + - return ValueError + """ + shutil.copy(src=os.path.join(TestRNUpdate.FILES_PATH, 'fake_pack_invalid/pack_metadata.json'), + dst=os.path.join(TestRNUpdate.FILES_PATH, 'fake_pack_invalid/_pack_metadata.json')) + from demisto_sdk.commands.update_release_notes.update_rn import UpdateRN + update_rn = UpdateRN(pack="HelloWorld", update_type='revision', pack_files={'HelloWorld'}) + update_rn.metadata_path = os.path.join(TestRNUpdate.FILES_PATH, 'fake_pack_invalid/pack_metadata.json') + self.assertRaises(ValueError, update_rn.bump_version_number) + os.remove(os.path.join(TestRNUpdate.FILES_PATH, 'fake_pack_invalid/pack_metadata.json')) + shutil.copy(src=os.path.join(TestRNUpdate.FILES_PATH, 'fake_pack_invalid/_pack_metadata.json'), + dst=os.path.join(TestRNUpdate.FILES_PATH, 'fake_pack_invalid/pack_metadata.json')) + + def test_bump_version_number_minor_overflow(self): + """ + Given: + - a pack name and a version before an overflow condition + When: + - bumping the version number in the metadata.json + Then: + - return ValueError + """ + shutil.copy(src=os.path.join(TestRNUpdate.FILES_PATH, 'fake_pack_invalid/pack_metadata.json'), + dst=os.path.join(TestRNUpdate.FILES_PATH, 'fake_pack_invalid/_pack_metadata.json')) + from demisto_sdk.commands.update_release_notes.update_rn import UpdateRN + update_rn = UpdateRN(pack="HelloWorld", update_type='minor', pack_files={'HelloWorld'}) + update_rn.metadata_path = os.path.join(TestRNUpdate.FILES_PATH, 'fake_pack_invalid/pack_metadata.json') + self.assertRaises(ValueError, update_rn.bump_version_number) + os.remove(os.path.join(TestRNUpdate.FILES_PATH, 'fake_pack_invalid/pack_metadata.json')) + shutil.copy(src=os.path.join(TestRNUpdate.FILES_PATH, 'fake_pack_invalid/_pack_metadata.json'), + dst=os.path.join(TestRNUpdate.FILES_PATH, 'fake_pack_invalid/pack_metadata.json')) + + def test_bump_version_number_major_overflow(self): + """ + Given: + - a pack name and a version before an overflow condition + When: + - bumping the version number in the metadata.json + Then: + - return ValueError + """ + shutil.copy(src=os.path.join(TestRNUpdate.FILES_PATH, 'fake_pack_invalid/pack_metadata.json'), + dst=os.path.join(TestRNUpdate.FILES_PATH, 'fake_pack_invalid/_pack_metadata.json')) + from demisto_sdk.commands.update_release_notes.update_rn import UpdateRN + update_rn = UpdateRN(pack="HelloWorld", update_type='major', pack_files={'HelloWorld'}) + update_rn.metadata_path = os.path.join(TestRNUpdate.FILES_PATH, 'fake_pack_invalid/pack_metadata.json') + self.assertRaises(ValueError, update_rn.bump_version_number) + os.remove(os.path.join(TestRNUpdate.FILES_PATH, 'fake_pack_invalid/pack_metadata.json')) + shutil.copy(src=os.path.join(TestRNUpdate.FILES_PATH, 'fake_pack_invalid/_pack_metadata.json'), + dst=os.path.join(TestRNUpdate.FILES_PATH, 'fake_pack_invalid/pack_metadata.json')) + + +class TestRNUpdateUnit: + FILES_PATH = os.path.normpath(os.path.join(__file__, f'{git_path()}/demisto_sdk/tests', 'test_files')) + + def test_ident_changed_file_type_integration(self, mocker): + """ + Given: + - a filepath of a changed file + When: + - determining the type of item changed (e.g. Integration, Script, Layout, etc.) + Then: + - return tuple where first value is the pack name, and second is the item type + """ + expected_result = ('VulnDB', 'Integration') + from demisto_sdk.commands.update_release_notes.update_rn import UpdateRN + update_rn = UpdateRN(pack="VulnDB", update_type='minor', pack_files={'HelloWorld'}) + filepath = os.path.join(TestRNUpdate.FILES_PATH, 'Integration/VulnDB/VulnDB.py') + mocker.patch.object(UpdateRN, 'find_corresponding_yml', return_value='Integrations/VulnDB/VulnDB.yml') + mocker.patch.object(UpdateRN, 'get_display_name', return_value='VulnDB') + result = update_rn.ident_changed_file_type(filepath) + assert expected_result == result + + def test_ident_changed_file_type_script(self, mocker): + """ + Given: + - a filepath of a changed file + When: + - determining the type of item changed (e.g. Integration, Script, Layout, etc.) + Then: + - return tuple where first value is the pack name, and second is the item type + """ + expected_result = ('VulnDB', 'Script') + from demisto_sdk.commands.update_release_notes.update_rn import UpdateRN + update_rn = UpdateRN(pack="VulnDB", update_type='minor', pack_files={'HelloWorld'}) + filepath = os.path.join(TestRNUpdate.FILES_PATH, 'Script/VulnDB/VulnDB.py') + mocker.patch.object(UpdateRN, 'find_corresponding_yml', return_value='Integrations/VulnDB/VulnDB.yml') + mocker.patch.object(UpdateRN, 'get_display_name', return_value='VulnDB') + result = update_rn.ident_changed_file_type(filepath) + assert expected_result == result + + def test_ident_changed_file_type_playbooks(self, mocker): + """ + Given: + - a filepath of a changed file + When: + - determining the type of item changed (e.g. Integration, Script, Layout, etc.) + Then: + - return tuple where first value is the pack name, and second is the item type + """ + expected_result = ('VulnDB', 'Playbook') + from demisto_sdk.commands.update_release_notes.update_rn import UpdateRN + update_rn = UpdateRN(pack="VulnDB", update_type='minor', pack_files={'HelloWorld'}) + filepath = os.path.join(TestRNUpdate.FILES_PATH, 'Playbooks/VulnDB/VulnDB_playbook.yml') + mocker.patch.object(UpdateRN, 'find_corresponding_yml', return_value='Integrations/VulnDB/VulnDB.yml') + mocker.patch.object(UpdateRN, 'get_display_name', return_value='VulnDB') + result = update_rn.ident_changed_file_type(filepath) + assert expected_result == result + + def test_ident_changed_file_type_incident_fields(self, mocker): + """ + Given: + - a filepath of a changed file + When: + - determining the type of item changed (e.g. Integration, Script, Layout, etc.) + Then: + - return tuple where first value is the pack name, and second is the item type + """ + expected_result = ('VulnDB', 'IncidentFields') + from demisto_sdk.commands.update_release_notes.update_rn import UpdateRN + update_rn = UpdateRN(pack="VulnDB", update_type='minor', pack_files={'HelloWorld'}) + filepath = os.path.join(TestRNUpdate.FILES_PATH, 'IncidentFields/VulnDB/VulnDB.json') + mocker.patch.object(UpdateRN, 'find_corresponding_yml', return_value='Integrations/VulnDB/VulnDB.yml') + mocker.patch.object(UpdateRN, 'get_display_name', return_value='VulnDB') + result = update_rn.ident_changed_file_type(filepath) + assert expected_result == result + + def test_ident_changed_file_type_incident_types(self, mocker): + """ + Given: + - a filepath of a changed file + When: + - determining the type of item changed (e.g. Integration, Script, Layout, etc.) + Then: + - return tuple where first value is the pack name, and second is the item type + """ + expected_result = ('VulnDB', 'IncidentTypes') + from demisto_sdk.commands.update_release_notes.update_rn import UpdateRN + update_rn = UpdateRN(pack="VulnDB", update_type='minor', pack_files={'HelloWorld'}) + filepath = os.path.join(TestRNUpdate.FILES_PATH, 'IncidentTypes/VulnDB/VulnDB.json') + mocker.patch.object(UpdateRN, 'find_corresponding_yml', return_value='Integrations/VulnDB/VulnDB.yml') + mocker.patch.object(UpdateRN, 'get_display_name', return_value='VulnDB') + result = update_rn.ident_changed_file_type(filepath) + assert expected_result == result + + def test_ident_changed_file_type_classifiers(self, mocker): + """ + Given: + - a filepath of a changed file + When: + - determining the type of item changed (e.g. Integration, Script, Layout, etc.) + Then: + - return tuple where first value is the pack name, and second is the item type + """ + expected_result = ('VulnDB', 'Classifiers') + from demisto_sdk.commands.update_release_notes.update_rn import UpdateRN + update_rn = UpdateRN(pack="VulnDB", update_type='minor', pack_files={'HelloWorld'}) + filepath = os.path.join(TestRNUpdate.FILES_PATH, 'Classifiers/VulnDB/VulnDB.json') + mocker.patch.object(UpdateRN, 'find_corresponding_yml', return_value='Integrations/VulnDB/VulnDB.yml') + mocker.patch.object(UpdateRN, 'get_display_name', return_value='VulnDB') + result = update_rn.ident_changed_file_type(filepath) + assert expected_result == result + + def test_ident_changed_file_type_layouts(self, mocker): + """ + Given: + - a filepath of a changed file + When: + - determining the type of item changed (e.g. Integration, Script, Layout, etc.) + Then: + - return tuple where first value is the pack name, and second is the item type + """ + expected_result = ('VulnDB', 'Layout') + from demisto_sdk.commands.update_release_notes.update_rn import UpdateRN + update_rn = UpdateRN(pack="VulnDB", update_type='minor', pack_files={'HelloWorld'}) + filepath = os.path.join(TestRNUpdate.FILES_PATH, 'Layouts/VulnDB/VulnDB.json') + mocker.patch.object(UpdateRN, 'find_corresponding_yml', return_value='Integrations/VulnDB/VulnDB.yml') + mocker.patch.object(UpdateRN, 'get_display_name', return_value='VulnDB') + result = update_rn.ident_changed_file_type(filepath) + assert expected_result == result + + def test_check_rn_directory(self): + """ + Given: + - a filepath for a release notes directory + When: + - determining if the directory exists + Then: + - create the directory if it does not exist + """ + from demisto_sdk.commands.update_release_notes.update_rn import UpdateRN + filepath = os.path.join(TestRNUpdate.FILES_PATH, 'ReleaseNotes') + update_rn = UpdateRN(pack="VulnDB", update_type='minor', pack_files={'HelloWorld'}) + update_rn.check_rn_dir(filepath) + + def test_create_markdown(self): + """ + Given: + - a filepath for a release notes file and a markdown string + When: + - creating a new markdown file + Then: + - create the file or skip if it exists. + """ + from demisto_sdk.commands.update_release_notes.update_rn import UpdateRN + update_rn = UpdateRN(pack="VulnDB", update_type='minor', pack_files={'HelloWorld'}) + filepath = os.path.join(TestRNUpdate.FILES_PATH, 'ReleaseNotes/1_1_1.md') + md_string = '### Test' + update_rn.create_markdown(release_notes_path=filepath, rn_string=md_string) diff --git a/demisto_sdk/commands/update_release_notes/update_rn.py b/demisto_sdk/commands/update_release_notes/update_rn.py new file mode 100644 index 00000000000..ba8498bb017 --- /dev/null +++ b/demisto_sdk/commands/update_release_notes/update_rn.py @@ -0,0 +1,212 @@ +""" +This script is used to create a release notes template +""" + +import errno +import json +import os +import sys + +from demisto_sdk.commands.common.constants import PACKS_PACK_META_FILE_NAME +from demisto_sdk.commands.common.hook_validations.structure import \ + StructureValidator +from demisto_sdk.commands.common.tools import (LOG_COLORS, get_json, + pack_name_to_path, print_color, + print_error, print_warning) + + +class UpdateRN: + def __init__(self, pack: str, update_type: str, pack_files: set, pre_release: bool = False): + + self.pack = pack + self.update_type = update_type + self.pack_meta_file = PACKS_PACK_META_FILE_NAME + self.pack_path = pack_name_to_path(self.pack) + self.metadata_path = os.path.join(self.pack_path, 'pack_metadata.json') + self.pack_files = pack_files + self.pre_release = pre_release + + def execute_update(self): + try: + new_version = self.bump_version_number(self.pre_release) + except ValueError as e: + print_error(e) + sys.exit(1) + rn_path = self.return_release_notes_path(new_version) + self.check_rn_dir(rn_path) + changed_files = {} + for packfile in self.pack_files: + fn, ft = self.ident_changed_file_type(packfile) + changed_files[fn] = ft + rn_string = self.build_rn_template(changed_files) + self.create_markdown(rn_path, rn_string) + + def _does_pack_metadata_exist(self): + """Check if pack_metadata.json exists""" + if not os.path.isfile(self.metadata_path): + print_error(f'"{self.metadata_path}" file does not exist, create one in the root of the pack') + return False + + return True + + def return_release_notes_path(self, input_version: str): + _new_version = input_version.replace('.', '_') + new_version = _new_version.replace('_prerelease', '') + return os.path.join(self.pack_path, 'ReleaseNotes', f'{new_version}.md') + + @staticmethod + def get_display_name(file_path): + struct = StructureValidator(file_path=file_path) + file_data = struct.load_data_from_file() + if 'name' in file_data: + name = file_data.get('name', None) + elif 'TypeName' in file_data: + name = file_data.get('TypeName', None) + else: + name = os.path.basename(file_path) + print_error(f"Could not find name in {file_path}") + # sys.exit(1) + return name + + @staticmethod + def find_corresponding_yml(file_path): + if file_path.endswith('.py'): + yml_filepath = file_path.replace('.py', '.yml') + else: + yml_filepath = file_path + return yml_filepath + + def ident_changed_file_type(self, file_path): + _file_type = None + file_name = 'N/A' + if self.pack in file_path: + _file_path = self.find_corresponding_yml(file_path) + file_name = self.get_display_name(_file_path) + if 'Playbooks' in file_path and ('TestPlaybooks' not in file_path): + _file_type = 'Playbook' + elif 'Integration' in file_path: + _file_type = 'Integration' + elif 'Script' in file_path: + _file_type = 'Script' + # incident fields and indicator fields are using the same scheme. + elif 'IncidentFields' in file_path: + _file_type = 'IncidentFields' + elif 'IncidentTypes' in file_path: + _file_type = 'IncidentTypes' + elif 'Classifiers' in file_path: + _file_type = 'Classifiers' + elif 'Layouts' in file_path: + _file_type = 'Layout' + + return file_name, _file_type + + def bump_version_number(self, pre_release: bool = False): + + new_version = None # This will never happen since we pre-validate the argument + data_dictionary = get_json(self.metadata_path) + if self.update_type == 'major': + version = data_dictionary.get('currentVersion', '99.99.99') + version = version.split('.') + version[0] = str(int(version[0]) + 1) + if int(version[0]) > 99: + raise ValueError(f"Version number is greater than 99 for the {self.pack} pack. " + f"Please verify the currentVersion is correct.") + version[1] = '0' + version[2] = '0' + new_version = '.'.join(version) + elif self.update_type == 'minor': + version = data_dictionary.get('currentVersion', '99.99.99') + version = version.split('.') + version[1] = str(int(version[1]) + 1) + if int(version[1]) > 99: + raise ValueError(f"Version number is greater than 99 for the {self.pack} pack. " + f"Please verify the currentVersion is correct. If it is, " + f"then consider bumping to a new Major version.") + version[2] = '0' + new_version = '.'.join(version) + # We validate the input via click + elif self.update_type == 'revision': + version = data_dictionary.get('currentVersion', '99.99.99') + version = version.split('.') + version[2] = str(int(version[2]) + 1) + if int(version[2]) > 99: + raise ValueError(f"Version number is greater than 99 for the {self.pack} pack. " + f"Please verify the currentVersion is correct. If it is, " + f"then consider bumping to a new Minor version.") + new_version = '.'.join(version) + if pre_release: + new_version = new_version + '_prerelease' + data_dictionary['currentVersion'] = new_version + + if self._does_pack_metadata_exist(): + with open(self.metadata_path, 'w') as fp: + json.dump(data_dictionary, fp, indent=4) + print_color(f"Updated pack metadata version at path : {self.metadata_path}", + LOG_COLORS.GREEN) + return new_version + + @staticmethod + def check_rn_dir(rn_path): + if not os.path.exists(os.path.dirname(rn_path)): + try: + os.makedirs(os.path.dirname(rn_path)) + except OSError as exc: # Guard against race condition + if exc.errno != errno.EEXIST: + raise + + def build_rn_template(self, changed_items: dict): + rn_string = '' + integration_header = False + playbook_header = False + script_header = False + inc_flds_header = False + classifier_header = False + layout_header = False + inc_types_header = False + for k, v in changed_items.items(): + if k == 'N/A': + continue + elif v == 'Integration': + if integration_header is False: + rn_string += '\n#### Integrations\n' + integration_header = True + rn_string += f'- __{k}__\n%%UPDATE_RN%%\n' + elif v == 'Playbook': + if playbook_header is False: + rn_string += '\n#### Playbooks\n' + playbook_header = True + rn_string += f'- __{k}__\n%%UPDATE_RN%%\n' + elif v == 'Script': + if script_header is False: + rn_string += '\n#### Scripts\n' + script_header = True + rn_string += f'- __{k}__\n%%UPDATE_RN%%\n' + elif v == 'IncidentFields': + if inc_flds_header is False: + rn_string += '\n#### IncidentFields\n' + inc_flds_header = True + rn_string += f'- __{k}__\n%%UPDATE_RN%%\n' + elif v == 'Classifiers': + if classifier_header is False: + rn_string += '\n#### Classifiers\n' + classifier_header = True + rn_string += f'- __{k}__\n%%UPDATE_RN%%\n' + elif v == 'Layouts': + if layout_header is False: + rn_string += '\n#### Layouts\n' + layout_header = True + rn_string += f'- __{k}__\n%%UPDATE_RN%%\n' + elif v == 'IncidentTypes': + if inc_types_header is False: + rn_string += '\n#### IncidentTypes\n' + inc_types_header = True + rn_string += f'- __{k}__\n%%UPDATE_RN%%\n' + return rn_string + + @staticmethod + def create_markdown(release_notes_path: str, rn_string: str): + if os.path.exists(release_notes_path): + print_warning(f"Release notes were found at {release_notes_path}. Skipping") + else: + with open(release_notes_path, 'w') as fp: + fp.write(rn_string) diff --git a/demisto_sdk/commands/validate/file_validator.py b/demisto_sdk/commands/validate/file_validator.py index 4387f97c0fc..d6d7a858144 100644 --- a/demisto_sdk/commands/validate/file_validator.py +++ b/demisto_sdk/commands/validate/file_validator.py @@ -210,7 +210,6 @@ def get_modified_and_added_files(self, tag='origin/master'): 'git diff --name-status {tag}..{compare_type}refs/heads/{branch}'.format(tag=tag, branch=self.branch_name, compare_type=compare_type)) - modified_files, added_files, _, old_format_files = self.get_modified_files( all_changed_files_string, tag=tag, diff --git a/demisto_sdk/tests/test_files/Integration/HelloWorld.py b/demisto_sdk/tests/test_files/Integration/HelloWorld.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/demisto_sdk/tests/test_files/Integration/HelloWorld.yml b/demisto_sdk/tests/test_files/Integration/HelloWorld.yml new file mode 100644 index 00000000000..e69de29bb2d diff --git a/demisto_sdk/tests/test_files/ReleaseNotes/1_1_1.md b/demisto_sdk/tests/test_files/ReleaseNotes/1_1_1.md new file mode 100644 index 00000000000..24d4a584678 --- /dev/null +++ b/demisto_sdk/tests/test_files/ReleaseNotes/1_1_1.md @@ -0,0 +1 @@ +### Test \ No newline at end of file diff --git a/demisto_sdk/tests/test_files/fake_pack/pack_metadata.json b/demisto_sdk/tests/test_files/fake_pack/pack_metadata.json index 2d088e90a66..7d836b38376 100644 --- a/demisto_sdk/tests/test_files/fake_pack/pack_metadata.json +++ b/demisto_sdk/tests/test_files/fake_pack/pack_metadata.json @@ -16,7 +16,10 @@ "deprecated": false, "certification": "certified", "useCases": [], - "keywords": ["AWS", "Feed"], + "keywords": [ + "AWS", + "Feed" + ], "price": 0, "dependencies": { "Base": { @@ -25,6 +28,6 @@ "name": "Base", "certification": "certified", "author": "Cortex XSOAR" - } + } } -} +} \ No newline at end of file diff --git a/demisto_sdk/tests/test_files/fake_pack_invalid/.pack-ignore b/demisto_sdk/tests/test_files/fake_pack_invalid/.pack-ignore new file mode 100644 index 00000000000..e69de29bb2d diff --git a/demisto_sdk/tests/test_files/fake_pack_invalid/.secrets-ignore b/demisto_sdk/tests/test_files/fake_pack_invalid/.secrets-ignore new file mode 100644 index 00000000000..c791d3c887c --- /dev/null +++ b/demisto_sdk/tests/test_files/fake_pack_invalid/.secrets-ignore @@ -0,0 +1 @@ +https://www.demisto.com \ No newline at end of file diff --git a/demisto_sdk/tests/test_files/fake_pack_invalid/Integrations/__init__.py b/demisto_sdk/tests/test_files/fake_pack_invalid/Integrations/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/demisto_sdk/tests/test_files/fake_pack_invalid/Integrations/fake_changelog/__init__.py b/demisto_sdk/tests/test_files/fake_pack_invalid/Integrations/fake_changelog/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/demisto_sdk/tests/test_files/fake_pack_invalid/README.md b/demisto_sdk/tests/test_files/fake_pack_invalid/README.md new file mode 100644 index 00000000000..e164476532b --- /dev/null +++ b/demisto_sdk/tests/test_files/fake_pack_invalid/README.md @@ -0,0 +1,2 @@ +boop +sade diff --git a/demisto_sdk/tests/test_files/fake_pack_invalid/__init__.py b/demisto_sdk/tests/test_files/fake_pack_invalid/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/demisto_sdk/tests/test_files/fake_pack_invalid/pack_metadata.json b/demisto_sdk/tests/test_files/fake_pack_invalid/pack_metadata.json new file mode 100644 index 00000000000..f521fc1e777 --- /dev/null +++ b/demisto_sdk/tests/test_files/fake_pack_invalid/pack_metadata.json @@ -0,0 +1,33 @@ +{ + "name": "AWS Feed", + "description": "Indicators feed from AWS", + "support": "Cortex XSOAR", + "serverMinVersion": "5.5.0", + "currentVersion": "99.99.99", + "author": "Cortex XSOAR", + "url": "https://www.paloaltonetworks.com/cortex", + "email": "", + "categories": [ + "Data Enrichment & Threat Intelligence" + ], + "tags": [], + "created": "2020-03-09T16:04:45Z", + "beta": false, + "deprecated": false, + "certification": "certified", + "useCases": [], + "keywords": [ + "AWS", + "Feed" + ], + "price": 0, + "dependencies": { + "Base": { + "mandatory": true, + "minVersion": "1.0.0", + "name": "Base", + "certification": "certified", + "author": "Cortex XSOAR" + } + } +} \ No newline at end of file